{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 包版本信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "numpy==1.13.1\n",
      "numpydoc==0.6.0\n",
      "pandas==0.20.3\n",
      "scikit-learn==0.19.0\n"
     ]
    }
   ],
   "source": [
    "!pip freeze | grep numpy\n",
    "!pip freeze | grep pandas\n",
    "!pip freeze | grep scikit-learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "## 隐藏warning信息\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "由于数据集以`txt`文件存储，使用`Pandas`的`read_table()`函数读入数据，分隔符为`\\t`。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15.26</td>\n",
       "      <td>14.84</td>\n",
       "      <td>0.8710</td>\n",
       "      <td>5.763</td>\n",
       "      <td>3.312</td>\n",
       "      <td>2.221</td>\n",
       "      <td>5.220</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14.88</td>\n",
       "      <td>14.57</td>\n",
       "      <td>0.8811</td>\n",
       "      <td>5.554</td>\n",
       "      <td>3.333</td>\n",
       "      <td>1.018</td>\n",
       "      <td>4.956</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14.29</td>\n",
       "      <td>14.09</td>\n",
       "      <td>0.9050</td>\n",
       "      <td>5.291</td>\n",
       "      <td>3.337</td>\n",
       "      <td>2.699</td>\n",
       "      <td>4.825</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13.84</td>\n",
       "      <td>13.94</td>\n",
       "      <td>0.8955</td>\n",
       "      <td>5.324</td>\n",
       "      <td>3.379</td>\n",
       "      <td>2.259</td>\n",
       "      <td>4.805</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16.14</td>\n",
       "      <td>14.99</td>\n",
       "      <td>0.9034</td>\n",
       "      <td>5.658</td>\n",
       "      <td>3.562</td>\n",
       "      <td>1.355</td>\n",
       "      <td>5.175</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      x1     x2      x3     x4     x5     x6     x7  label\n",
       "0  15.26  14.84  0.8710  5.763  3.312  2.221  5.220      1\n",
       "1  14.88  14.57  0.8811  5.554  3.333  1.018  4.956      1\n",
       "2  14.29  14.09  0.9050  5.291  3.337  2.699  4.825      1\n",
       "3  13.84  13.94  0.8955  5.324  3.379  2.259  4.805      1\n",
       "4  16.14  14.99  0.9034  5.658  3.562  1.355  5.175      1"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_table('seed.txt', sep='\\t', names = ['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7','label'])\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "然后，我们将`data['label']`的取值减`1`, 标签取值范围为`[0, 1, 2]`。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15.26</td>\n",
       "      <td>14.84</td>\n",
       "      <td>0.8710</td>\n",
       "      <td>5.763</td>\n",
       "      <td>3.312</td>\n",
       "      <td>2.221</td>\n",
       "      <td>5.220</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14.88</td>\n",
       "      <td>14.57</td>\n",
       "      <td>0.8811</td>\n",
       "      <td>5.554</td>\n",
       "      <td>3.333</td>\n",
       "      <td>1.018</td>\n",
       "      <td>4.956</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14.29</td>\n",
       "      <td>14.09</td>\n",
       "      <td>0.9050</td>\n",
       "      <td>5.291</td>\n",
       "      <td>3.337</td>\n",
       "      <td>2.699</td>\n",
       "      <td>4.825</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13.84</td>\n",
       "      <td>13.94</td>\n",
       "      <td>0.8955</td>\n",
       "      <td>5.324</td>\n",
       "      <td>3.379</td>\n",
       "      <td>2.259</td>\n",
       "      <td>4.805</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16.14</td>\n",
       "      <td>14.99</td>\n",
       "      <td>0.9034</td>\n",
       "      <td>5.658</td>\n",
       "      <td>3.562</td>\n",
       "      <td>1.355</td>\n",
       "      <td>5.175</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      x1     x2      x3     x4     x5     x6     x7  label\n",
       "0  15.26  14.84  0.8710  5.763  3.312  2.221  5.220      0\n",
       "1  14.88  14.57  0.8811  5.554  3.333  1.018  4.956      0\n",
       "2  14.29  14.09  0.9050  5.291  3.337  2.699  4.825      0\n",
       "3  13.84  13.94  0.8955  5.324  3.379  2.259  4.805      0\n",
       "4  16.14  14.99  0.9034  5.658  3.562  1.355  5.175      0"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['label'] = data['label'] - 1\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(210, 8)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "x1       0\n",
       "x2       0\n",
       "x3       0\n",
       "x4       0\n",
       "x5       0\n",
       "x6       0\n",
       "x7       0\n",
       "label    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 检查有无缺失值\n",
    "np.sum(data.isnull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    70\n",
       "1    70\n",
       "0    70\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 标签label的取值1，2，3\n",
    "data['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_label = np.unique(data['label'])\n",
    "class_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 210 entries, 0 to 209\n",
      "Data columns (total 8 columns):\n",
      "x1       210 non-null float64\n",
      "x2       210 non-null float64\n",
      "x3       210 non-null float64\n",
      "x4       210 non-null float64\n",
      "x5       210 non-null float64\n",
      "x6       210 non-null float64\n",
      "x7       210 non-null float64\n",
      "label    210 non-null int64\n",
      "dtypes: float64(7), int64(1)\n",
      "memory usage: 13.2 KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>210.000000</td>\n",
       "      <td>210.000000</td>\n",
       "      <td>210.000000</td>\n",
       "      <td>210.000000</td>\n",
       "      <td>210.000000</td>\n",
       "      <td>210.000000</td>\n",
       "      <td>210.000000</td>\n",
       "      <td>210.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>14.847524</td>\n",
       "      <td>14.559286</td>\n",
       "      <td>0.870999</td>\n",
       "      <td>5.628533</td>\n",
       "      <td>3.258605</td>\n",
       "      <td>3.700201</td>\n",
       "      <td>5.408071</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.909699</td>\n",
       "      <td>1.305959</td>\n",
       "      <td>0.023629</td>\n",
       "      <td>0.443063</td>\n",
       "      <td>0.377714</td>\n",
       "      <td>1.503557</td>\n",
       "      <td>0.491480</td>\n",
       "      <td>0.818448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>10.590000</td>\n",
       "      <td>12.410000</td>\n",
       "      <td>0.808100</td>\n",
       "      <td>4.899000</td>\n",
       "      <td>2.630000</td>\n",
       "      <td>0.765100</td>\n",
       "      <td>4.519000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>12.270000</td>\n",
       "      <td>13.450000</td>\n",
       "      <td>0.856900</td>\n",
       "      <td>5.262250</td>\n",
       "      <td>2.944000</td>\n",
       "      <td>2.561500</td>\n",
       "      <td>5.045000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>14.355000</td>\n",
       "      <td>14.320000</td>\n",
       "      <td>0.873450</td>\n",
       "      <td>5.523500</td>\n",
       "      <td>3.237000</td>\n",
       "      <td>3.599000</td>\n",
       "      <td>5.223000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>17.305000</td>\n",
       "      <td>15.715000</td>\n",
       "      <td>0.887775</td>\n",
       "      <td>5.979750</td>\n",
       "      <td>3.561750</td>\n",
       "      <td>4.768750</td>\n",
       "      <td>5.877000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>21.180000</td>\n",
       "      <td>17.250000</td>\n",
       "      <td>0.918300</td>\n",
       "      <td>6.675000</td>\n",
       "      <td>4.033000</td>\n",
       "      <td>8.456000</td>\n",
       "      <td>6.550000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               x1          x2          x3          x4          x5          x6  \\\n",
       "count  210.000000  210.000000  210.000000  210.000000  210.000000  210.000000   \n",
       "mean    14.847524   14.559286    0.870999    5.628533    3.258605    3.700201   \n",
       "std      2.909699    1.305959    0.023629    0.443063    0.377714    1.503557   \n",
       "min     10.590000   12.410000    0.808100    4.899000    2.630000    0.765100   \n",
       "25%     12.270000   13.450000    0.856900    5.262250    2.944000    2.561500   \n",
       "50%     14.355000   14.320000    0.873450    5.523500    3.237000    3.599000   \n",
       "75%     17.305000   15.715000    0.887775    5.979750    3.561750    4.768750   \n",
       "max     21.180000   17.250000    0.918300    6.675000    4.033000    8.456000   \n",
       "\n",
       "               x7       label  \n",
       "count  210.000000  210.000000  \n",
       "mean     5.408071    1.000000  \n",
       "std      0.491480    0.818448  \n",
       "min      4.519000    0.000000  \n",
       "25%      5.045000    0.000000  \n",
       "50%      5.223000    1.000000  \n",
       "75%      5.877000    2.000000  \n",
       "max      6.550000    2.000000  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分类建模\n",
    "\n",
    "\n",
    "### 逻辑回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model.logistic import LogisticRegression\n",
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(pred,test_y):\n",
    "    \n",
    "    import seaborn as sns\n",
    "    import matplotlib.pyplot as plt\n",
    "    import sklearn.metrics as metrics\n",
    "    # 输出分类的准确率\n",
    "    print(\"Accuracy: %.4f\"  % (metrics.accuracy_score(test_y,pred)))\n",
    "    \n",
    "    # 输出衡量分类效果的各项指标\n",
    "    print(classification_report(test_y, pred)) \n",
    "    \n",
    "    # 更直观的，我们通过seaborn画出混淆矩阵\n",
    "    %matplotlib inline\n",
    "    plt.figure(figsize=(6,4))\n",
    "    colorMetrics = metrics.confusion_matrix(test_y,pred)\n",
    "    \n",
    "    # 坐标y代表test_y，即真实的类别，坐标x代表估计出的类别pred\n",
    "    sns.heatmap(colorMetrics, annot=True, fmt='d', xticklabels=[1,2,3],yticklabels=[1,2,3])\n",
    "    sns.plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 数据准备\n",
    "\n",
    "X = data.iloc[:, :-1]\n",
    "y = data['label']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'C': [0.001, 0.01, 0.1, 1, 10]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LogisticRegression()\n",
    "param_dict = {'C': [0.001, 0.01, 0.1, 1, 10]}\n",
    "gs = GridSearchCV(lr, param_grid = param_dict, scoring='accuracy')\n",
    "gs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_C</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split1_train_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.015219</td>\n",
       "      <td>0.002470</td>\n",
       "      <td>0.382166</td>\n",
       "      <td>0.381955</td>\n",
       "      <td>0.001</td>\n",
       "      <td>{u'C': 0.001}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.415094</td>\n",
       "      <td>0.355769</td>\n",
       "      <td>0.377358</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.352941</td>\n",
       "      <td>0.415094</td>\n",
       "      <td>0.019821</td>\n",
       "      <td>0.003011</td>\n",
       "      <td>0.025520</td>\n",
       "      <td>0.024714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.001227</td>\n",
       "      <td>0.000423</td>\n",
       "      <td>0.783439</td>\n",
       "      <td>0.764393</td>\n",
       "      <td>0.01</td>\n",
       "      <td>{u'C': 0.01}</td>\n",
       "      <td>4</td>\n",
       "      <td>0.754717</td>\n",
       "      <td>0.778846</td>\n",
       "      <td>0.867925</td>\n",
       "      <td>0.759615</td>\n",
       "      <td>0.725490</td>\n",
       "      <td>0.754717</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>0.000113</td>\n",
       "      <td>0.061473</td>\n",
       "      <td>0.010414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.002254</td>\n",
       "      <td>0.000865</td>\n",
       "      <td>0.885350</td>\n",
       "      <td>0.882196</td>\n",
       "      <td>0.1</td>\n",
       "      <td>{u'C': 0.1}</td>\n",
       "      <td>3</td>\n",
       "      <td>0.886792</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>0.905660</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>0.862745</td>\n",
       "      <td>0.877358</td>\n",
       "      <td>0.000599</td>\n",
       "      <td>0.000528</td>\n",
       "      <td>0.017491</td>\n",
       "      <td>0.003421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.002277</td>\n",
       "      <td>0.000545</td>\n",
       "      <td>0.910828</td>\n",
       "      <td>0.920356</td>\n",
       "      <td>1</td>\n",
       "      <td>{u'C': 1}</td>\n",
       "      <td>2</td>\n",
       "      <td>0.886792</td>\n",
       "      <td>0.932692</td>\n",
       "      <td>0.962264</td>\n",
       "      <td>0.903846</td>\n",
       "      <td>0.882353</td>\n",
       "      <td>0.924528</td>\n",
       "      <td>0.000724</td>\n",
       "      <td>0.000129</td>\n",
       "      <td>0.036763</td>\n",
       "      <td>0.012140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.001934</td>\n",
       "      <td>0.000510</td>\n",
       "      <td>0.949045</td>\n",
       "      <td>0.949020</td>\n",
       "      <td>10</td>\n",
       "      <td>{u'C': 10}</td>\n",
       "      <td>1</td>\n",
       "      <td>0.943396</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.913462</td>\n",
       "      <td>0.901961</td>\n",
       "      <td>0.952830</td>\n",
       "      <td>0.000550</td>\n",
       "      <td>0.000198</td>\n",
       "      <td>0.040093</td>\n",
       "      <td>0.027610</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  mean_score_time  mean_test_score  mean_train_score param_C  \\\n",
       "0       0.015219         0.002470         0.382166          0.381955   0.001   \n",
       "1       0.001227         0.000423         0.783439          0.764393    0.01   \n",
       "2       0.002254         0.000865         0.885350          0.882196     0.1   \n",
       "3       0.002277         0.000545         0.910828          0.920356       1   \n",
       "4       0.001934         0.000510         0.949045          0.949020      10   \n",
       "\n",
       "          params  rank_test_score  split0_test_score  split0_train_score  \\\n",
       "0  {u'C': 0.001}                5           0.415094            0.355769   \n",
       "1   {u'C': 0.01}                4           0.754717            0.778846   \n",
       "2    {u'C': 0.1}                3           0.886792            0.884615   \n",
       "3      {u'C': 1}                2           0.886792            0.932692   \n",
       "4     {u'C': 10}                1           0.943396            0.980769   \n",
       "\n",
       "   split1_test_score  split1_train_score  split2_test_score  \\\n",
       "0           0.377358            0.375000           0.352941   \n",
       "1           0.867925            0.759615           0.725490   \n",
       "2           0.905660            0.884615           0.862745   \n",
       "3           0.962264            0.903846           0.882353   \n",
       "4           1.000000            0.913462           0.901961   \n",
       "\n",
       "   split2_train_score  std_fit_time  std_score_time  std_test_score  \\\n",
       "0            0.415094      0.019821        0.003011        0.025520   \n",
       "1            0.754717      0.000030        0.000113        0.061473   \n",
       "2            0.877358      0.000599        0.000528        0.017491   \n",
       "3            0.924528      0.000724        0.000129        0.036763   \n",
       "4            0.952830      0.000550        0.000198        0.040093   \n",
       "\n",
       "   std_train_score  \n",
       "0         0.024714  \n",
       "1         0.010414  \n",
       "2         0.003421  \n",
       "3         0.012140  \n",
       "4         0.027610  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gs.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 10}\n",
      "0.949044585987\n"
     ]
    }
   ],
   "source": [
    "print gs.best_params_\n",
    "print gs.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.942679738562\n"
     ]
    }
   ],
   "source": [
    "lr_cls = gs.best_estimator_\n",
    "score = cross_val_score(lr_cls, X_train, y_train, cv = 10, scoring = 'accuracy')\n",
    "print np.mean(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9811\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      1.00      0.97        16\n",
      "          1       1.00      0.94      0.97        18\n",
      "          2       1.00      1.00      1.00        19\n",
      "\n",
      "avg / total       0.98      0.98      0.98        53\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD3CAYAAABYUUzPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD/tJREFUeJzt3XuwVeV5x/Hv3oebHsB6iRqrkQyjb2xBZMqIIioWnVij\n0wTbxkQ0ahRvOGpqEbyQeivBK5HaajGoSXGcGokmMzHGqIAX4qAmR4j4Rm3Fa7zfAEWP5/QPTwhq\nPNf97rV49/fjrJmzNqzFw5rxx3Oe9a51Ku3t7UiSaq9adAGSlCsDVpISMWAlKREDVpISMWAlKZF+\nKU8+eewUlygkNv/OWUWXINXEgKFbVvp6jl133LfbmfPoqsV9/vO6YgcrSYkk7WAlqZ4qleRNaY8Y\nsJKyUamU65tyA1ZSNqrYwUpSEo4IJCmRqiMCSUqjbB1sueJekjJiByspG02VpqJL+BgDVlI2yjYi\nMGAlZaNasoB1BitJidjBSspGpWQ9owErKRtNVQNWkpKolOxR2XLFvSRlxA5WUjZ8VFaSEnEdrCQl\nUrZ1sAaspGyU7SaXASspG85gJSkRZ7CSlIgzWElKxBmsJCXiiECSEnFEIEmJOCKQpETKtkyrXNVI\nUkbsYCVlw5tckpRIU8lGBAaspGzUehVBCGEsMDvGOCGEsDUwD9gcaAKOjDE+1dnxDR+ww//6ixx2\n8iQuOukyhm4+hG+fdQTNQzalWq1y9XnX8fLzrxRdYjba2tq4cPalxCeeYED/AZx3zgy+sMP2RZeV\nFa9x7YQQpgFHAGs6ProYWBBj/J8Qwn7Al4BOA7Zc/XSdfWXylzn2rCPpP7A/AIdNPZQHfvEgF55w\nKTdfcxvbDdu24ArzcveiJaxb9z4L5s/jtKkncsmcK4suKTuNfo0rlUq3t254Cpi0wf5ewPYhhF8B\nhwOLujpBQwfsy8+/wpzp/7l+f+dRw9li682ZPvd09vry7qx8OBZYXX4eaWlh/LixAIwaOYLHVj5e\ncEX5afRrXK1Uur11JcZ4C/DBBh8NA96IMe4PPAOc2WU9vfx7ZGHZPY/wYeuH6/e3+vxWrHlnLd87\n5Qpefel1Dj7ywAKry8+aNWsZ3Dx4/X612kRra2uBFeWn0a9xpQf/9cJrwE87vv4ZMKarAxo6YD9p\n9VureWRJCwC/ufdRvrjLjgVXlJfm5k1Zs3bt+v229jb69Wv42wA11ejXuJYd7J9xH3BQx9f7AL/r\n6oBOr3wI4R5g4Cc+rgDtMcZxvamwzH7f8iSj9hrJ/bf/mi+N3onn//eFokvKyuhRu7Joyf0ceMBE\nWpavYKfhw4suKTuNfo0Tr4P9Z+DaEMKJwFvAN7s6oKt/2qbz0bKErwHZf5+x4Ps3c+xZRzJx0r68\nu/pdrpp5bdElZWXihH1Z+uAyJh8zhXbauWDm2UWXlJ1Gv8a1XqYVY3wa2KPj61XAAT05vtLe3t7p\nbwgh/AvwZIzxJz0tbvLYKZ2fXH02/85ZRZcg1cSAoVv2OR2/Pe7kbmfODx64KvljX10OZ2KMl6Qu\nQpJqwdcVSlIivotAkhKxg5WkROxgJSmRsv1EAx80kKRE7GAlZaNargbWgJWUj6Zqub4pN2AlZaNs\nN7nKFfeSlBE7WEnZqJZsFYEBKykbZRsRGLCSsuGTXJKUSMny1YCVlA87WElKpGyPyhqwkrLhTS5J\nSsQRgSQlUrJ8NWAl5cMOVpIS8SaXJCViBytJiZQsX32bliSlYgcrKRu+cFuSEinbiMCAlZSNst3k\nKlc/LUkZsYOVlA3XwUpSIr7sRZISaaqWK2CdwUpSIkk72GsWTk95egGT9j6l6BKyt/DeuUWXoG5y\nRCBJiZRsQmDASsqHHawkJVKyfDVgJeWjbE9yGbCSslHrBw1CCGOB2THGCSGE3YC5wIfAOuDIGONL\nnR3vMi1J2ahUur91JYQwDbgWGNTx0feBU2KME4CFwJldncOAlZSNaqXS7a0bngImbbB/WIzxtx1f\n9wPe67Kenv8VJCl/McZbgA822H8RIIQwDpgKXNHVOZzBSspGNfFC2BDC14Gzga/EGF/p6vcbsJKy\nkXIdbAhhMnA8MCHG+Hp3jjFgJWUjVQMbQmgCrgSeARaGEAAWxxi/29lxBqwkfYYY49PAHh27W/T0\neANWUjZ8VFaSEinb+2ANWEnZKFsH6zpYSUrEDlZSNkrWwBqwkvJRthGBASspGyXLVwNWUj58H6wk\nJVKyfDVgJeXDGawkJVKyfDVgJeWjbB2sDxpIUiJ2sJKy4bsIJCmRkk0IDFhJ+XAGK0kNwg5WUjZK\n1sAasJLykfqnyvaUAbuB5Ssf58r/ms+8Ky4uupTs7DxyOEef+g1mHHsh02afwuZbbgbA1tt9jrj8\nSS4+c27BFeahra2NC2dfSnziCQb0H8B558zgCztsX3RZdVO2GawB2+H6m27m53fezaBBA4suJTuH\nHnUw+x08nvfeXQewPkybhzQz69qzmXfJj4osLyt3L1rCunXvs2D+PFqWr+CSOVcy9zIbhqL0+CZX\nCCHLBNphu89z6XnnFF1Gll589iX+7TtzPvX54Scdys9u+iVvvPpmAVXl6ZGWFsaPGwvAqJEjeGzl\n4wVXVF+VSve3evjMgA0hHBJCWBVCeDKE8PUNfun2OtRVdxP3GU+/fjb0KTxw1zJaW1s/9tlmWwxl\n1NgR3HXb4oKqytOaNWsZ3Dx4/X612vSpa5+zSqXS7a0eOutgzwZ2A8YCx4cQvtXxebmGHNoo7bX/\n7iz++f20tbUXXUpWmps3Zc3atev329rbGqpx2Gg6WOD9GOMbMcbXgL8HpoYQ9gP8P0J9ttseI3j4\nvpaiy8jO6FG7cu/9SwFoWb6CnYYPL7ii+mqqVrq91UNn/7Q9HUK4HDg3xvhOCGEScAfwF3WpTFn7\ny2Hb8YfnXy66jOxMnLAvSx9cxuRjptBOOxfMPLvokhpaZwF7DDCZjo41xvhsRwc7ox6FFWG7bbfh\nh1d9+maM+u7lF17ljCO+u37/5EnTCqwmX9VqlZkzGvfabjTLtGKMrcD1n/jsJeC0xDVJUq+ULF9d\nByspHxWf5JKkNOxgJSmRjWYGK0kbm5LlqwErKR92sJKUSMny1Z9oIEmp2MFKykalWq6e0YCVlI2y\njQgMWEnZ8EEDSSq5EEJ/4AZgGPAhcFyMscdvLy/XwEKS+qCG74M9COgXYxwHnA9c1Jt67GAlZaOG\n62B/D/QLIVSBocAHvTmJASspGzX8sd2r+Wg88DiwFXBwr+qpVTWSlJHTgTtijDsDo4AbQgiDenoS\nO1hJ2ajhMq03+NNY4HWgP9DU05MYsJKyUcMZ7BXA/BDCvcAA4KwY45qensSAlZSPGg09Y4yrgX/q\n63kMWEnZ8G1akpRIyfLVgJWUDztYSUqkZPlqwErKSMkS1gcNJCkRO1hJ2ag2lauDNWAlZcObXJKU\nSMny1RmsJKViByspHyVrYQ1YSdnwZ3JJUiJlC1hnsJKUSNIOtv+QzVKeXsDCe+cWXUL2xoycVHQJ\nDeHRVYv7fI6SjWAdEUjKR9lGBAaspGz4oIEkpVKufDVgJeXDDlaSEjFgJSmVki08NWAlZaNsHWzJ\n8l6S8mEHKykbroOVpEQMWElKxRmsJDUGO1hJ2ShZA2vASspH2ZZpGbCSslFpKtfUs1zVSFJG7GAl\n5aNcEwIDVlI+nMFKUiI+aCBJiVSq5bqtVK5qJCkjdrCS8lGuCYEBKykftZ7BhhC2Bh4GDogxPt7T\n4w1YSfmo4SqCEEJ/4Brg3d6ewxmspGxUKpVub91wKXA18EJv6zFgJeWjWun+1okQwlHAKzHGO/pU\nTl8OlqQyqWEHewxwQAhhEbAb8MMQwrY9rccZrKR81GgEG2Pc549fd4TsCTHGP/T0PAaspGz4qKwk\nbURijBN6e6wBKykfJXsXgTe5gLa2Ns6fdTGHH3McRx9/Ms88+1zRJWXJ65zWyN124Qc3zQFglxE7\nseC2q7n+5rlMP+/U0n3rnEqlWu32Vg8GLHD3oiWsW/c+C+bP47SpJ3LJnCuLLilLXud0jj7+G/zr\n7GkMHDgAgJmzzuDi8+dy1D+ewup3VnPQV/cvuML6qPE62D7rUcCGEDYJIQxMVUxRHmlpYfy4sQCM\nGjmCx1b2+Ik4dYPXOZ1nn3me048/Z/3+Ntt+jpaHfwfAbx9awegxI4sqraF1GrAhhL8KIdwaQrgu\nhLA/sBJ4LIRwcH3Kq481a9YyuHnw+v1qtYnW1tYCK8qT1zmdX92+hNbWD9fvP/fsi/zN2FEA7Lv/\nODbZdJOiSquvGj1oULNyuvj1q4ErgEXAj4HdgdHAjLRl1Vdz86asWbt2/X5bexv9+nn/r9a8zvUz\n84zvcexJhzPvxst5/dU3ePP1N4suqS42thFBNca4OMZ4A3BrjPHlGOPbQFZtx+hRu3Lv/UsBaFm+\ngp2GDy+4ojx5netn77/dk+mnXshx3/wOm22+GUvve6jokuqi0lTt9lYPXbUPMYRwLTAlxngUQAhh\nOtDjJxrKbOKEfVn64DImHzOFdtq5YObZRZeUJa9z/Tzzf88x78bLee+9dSxb+hvuu+fBoktqSJX2\n9vbP/MUQQhU4JMZ42wafTQYWxhjXfuaBHd5/+7XPPrm0kRgzclLRJTSER1ct7vP37a8+tLTbmbPV\nmD2Tzwk67WBjjG3AbZ/47L+TViRJvVS29b7eYZCUDwNWktLwx3ZLUip2sJKUiAErSWl4k0uSUinZ\nDNa3aUlSInawkrJRqZSrZzRgJWWjXi/S7i4DVlI+nMFKUmOwg5WUDZdpSVIqBqwkpVFpaiq6hI9x\nBitJidjBSsqHIwJJSsObXJKUik9ySVIavnBbklJxRCBJaTiDlaRUnMFKUiIlm8GWK+4lKSN2sJKy\n4QxWkhKpVMv1LgIDVlI+SnaTq1zVSFJG7GAlZaNWT3KFEKrAfwCjgHXAsTHGJ3t6HjtYSfmoVLq/\nde6rwKAY457AdOCy3pRjwErKRqXa1O2tC+OBXwDEGH8NjOlNPUlHBAOGblmuNRNSLzy6anHRJaib\napg5Q4G3Ntj/MITQL8bY2pOT2MFK0qe9DQzZYL/a03AFA1aS/pz7gYMAQgh7AMt7cxJXEUjSp/0E\nOCCE8ABQAY7uzUkq7e3tNa1KkvQRRwSSlIgBK0mJGLCSlIg3uTYQQhgLzI4xTii6lhyFEPoD84Fh\nwEDgwhjjTwstKjMhhCZgHhCAduCEGOOKYqtqXHawHUII04BrgUFF15KxycBrMca9gQOBfy+4nhwd\nAhBj3As4B7io2HIamwH7J08Bk4ouInM3A+d2fF0BerxwW52LMd4KTOnY3RF4s8ByGp4jgg4xxltC\nCMOKriNnMcbVACGEIcCP+ajDUo3FGFtDCDcAXwP+oeh6GpkdrOoqhLADcA/woxjjjUXXk6sY47eA\nnYF5IYTmoutpVHawqpsQwjbAL4GpMca7iq4nRyGEI4DtY4yzgLVAW8emAhiwqqezgM2Bc0MIf5zF\n/l2M8d0Ca8rNQuC6EMISoD9wmte3OD4qK0mJOIOVpEQMWElKxICVpEQMWElKxICVpEQMWElKxICV\npET+Hw+2hNghxlWWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11805f590>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(y_test, lr_cls.predict(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 高斯混合模型\n",
    "\n",
    "#### 建立模型\n",
    "\n",
    "在本案例中，我们使用`Python`语言实现高斯混合模型。我们把实现过程分为两个部分\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ 计算Expectation步\n",
    "\n",
    "在这一步，我们计算每条数据记录$x_{n}$分属K个标签的概率$\\gamma_{n}(z_{k}) = p(z_{k} = 1 \\vert x_{n}, \\theta)$。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''expectation_step函数\n",
    "\n",
    "输入参数：X: 数据集\n",
    "        pis: p(z = 1)的概率\n",
    "        mus: 高斯分布的均值\n",
    "        sigmas: 高斯分布的协方差\n",
    "\n",
    "返回参数： $x_{n}$分属K个标签的概率矩阵\n",
    "\n",
    "'''\n",
    "def expectation_step(X, pis, mus, sigmas):\n",
    "    \n",
    "    import numpy as np\n",
    "    from scipy.stats import multivariate_normal\n",
    "    \n",
    "    gammas  = np.array([value[0]*multivariate_normal(mean = value[1], cov = value[2]).pdf(X) for value in zip(pis, mus, sigmas)])\n",
    "    return gammas / gammas.sum(axis = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ 计算Maximization步\n",
    "\n",
    "在这一步，我们以完全数据的对数极大似然函数为目标，更新参数`pis`, `mus`, `sigmas`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''maximization_step函数\n",
    "\n",
    "输入参数：X: 数据集\n",
    "        gammas：$x_{n}$分属K个标签的概率矩阵\n",
    "        pis: p(z = 1)的概率\n",
    "        mus: 高斯分布的均值\n",
    "        sigmas: 高斯分布的协方差\n",
    "'''\n",
    "\n",
    "def maximization_step(X, gammas, pis, mus, sigmas):\n",
    "    \n",
    "    # 更新pis\n",
    "    pis = gammas.sum(axis = 1)  / gammas.shape[1]\n",
    "\n",
    "    \n",
    "    # 更新mus\n",
    "    mus = np.multiply(1.0/gammas.sum(axis = 1).reshape(-1, 1), np.dot(gammas, X))\n",
    "    \n",
    "    \n",
    "    # 更新sigmas\n",
    "    \n",
    "    for k in range(len(sigmas)):\n",
    "      \n",
    "            \n",
    "        centered_X = X - mus[k]\n",
    "            \n",
    "        sigmas[k] = np.dot(np.multiply(gammas[k], centered_X.T), centered_X) / gammas[k].sum()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "更新最大似然函数的下界`lower_bound`。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''cal_lower_bound函数\n",
    "\n",
    "输入参数：X: 数据集\n",
    "        gammas：$x_{n}$分属K个标签的概率矩阵\n",
    "        pis: p(z = 1)的概率\n",
    "        mus: 训练得到的高斯分布的均值\n",
    "        sigmas: 训练得到的高斯分布的协方差\n",
    "\n",
    "返回参数：对数最大似然函数的下界\n",
    "\n",
    "'''\n",
    "\n",
    "def cal_lower_bound(X, gammas, pis, mus, sigmas):\n",
    "    \n",
    "    from scipy.stats import multivariate_normal\n",
    "\n",
    "    lower_bound = 0\n",
    "        \n",
    "    for gamma, pi, mu, sigma in zip(gammas, pis, mus, sigmas):\n",
    "    \n",
    "        lower_bound  += np.multiply(gamma, np.log(pi*multivariate_normal(mu, sigma).pdf(X)))\n",
    "\n",
    "    return lower_bound.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来，我们实现`train_GMM`函数，计算各个高斯分布的统计特征。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''train_GMM函数\n",
    "\n",
    "输入参数：X: 数据集\n",
    "        k: 高斯分布的个数\n",
    "        max_iter: 迭代的最高次数\n",
    "        alpha: 最大似然函数下界收敛的阈值\n",
    "        \n",
    "        \n",
    "返回参数：(mus, sigmas)以元组形式返回参数\n",
    "        mus: 高斯分布的均值 K times d\n",
    "        sigmas：高斯分布的协方差 d times d\n",
    "        \n",
    "'''\n",
    "def train_GMM(X, K, max_iter, alpha):\n",
    "\n",
    "    indicator = 0\n",
    "    lower_bound = 0\n",
    "   \n",
    "    # 初始化\n",
    "    N, d = X.shape\n",
    "    d = d - 1\n",
    "\n",
    "    # 初始化pi，一维数组，即 K times 1的向量\n",
    "    pis = np.random.random(K)\n",
    "    pis /= pis.sum()\n",
    "\n",
    "    # 初始化mu，二维数组，即 K times d的矩阵\n",
    "    #mus = np.random.random((K, d))\n",
    "    mus = np.array([X[X['label'] == num].mean()[:-1] for num in range(K)])\n",
    "\n",
    "    # 初始化sigmas  三维数组， 即长度为K，元素为 d times d 矩阵的数组\n",
    "    sigmas = np.array([np.eye(d)] * K)\n",
    "    \n",
    "    X_hat = X.iloc[:,:-1].values\n",
    "    while indicator <= max_iter:\n",
    "    \n",
    "        lower_bound_old = lower_bound\n",
    "    \n",
    "        # E-step\n",
    "        gammas = expectation_step(X_hat, pis, mus, sigmas)\n",
    "        \n",
    "        # M-step\n",
    "        maximization_step(X_hat, gammas, pis, mus, sigmas)\n",
    "        \n",
    "        # Update 下界\n",
    "        lower_bound = cal_lower_bound(X_hat, gammas, pis, mus, sigmas)\n",
    "    \n",
    "        # 检查终止条件\n",
    "        if abs(lower_bound - lower_bound_old) < alpha:\n",
    "            print 'reach stopping criterion!'\n",
    "            break\n",
    "    \n",
    "        # 更新迭代次数\n",
    "        indicator += 1\n",
    "    return (mus, sigmas)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "计算高斯分布的参数之后，我们需要判定样本来自于各个分布的概率大小。也就是说将样本代入分布的密度函数，计算其密度，取最大值所在的分布作为该样本的总体分布，并将样本归属于样本所在的类。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''predict函数\n",
    "\n",
    "输入参数： X：数据集\n",
    "        mus: 各个高斯分布的均值  K time d\n",
    "        sigmas: 各个高斯分布的协方差   K times 1 元素为d times d 矩阵\n",
    "\n",
    "输出参数：数据集的各个数据记录所对应的标签(如0,1,2)\n",
    "\n",
    "'''\n",
    "\n",
    "def predict(X, mus, sigmas):\n",
    "    \n",
    "    from scipy.stats import multivariate_normal\n",
    "\n",
    "    labels = np.zeros(X.shape[0])\n",
    "    \n",
    "    for index, item in enumerate(X.iloc[:,:-1].values):\n",
    "    \n",
    "        probs = [multivariate_normal(mu, sigma).pdf(item) for mu, sigma in zip(mus, sigmas)]\n",
    "    \n",
    "        labels[index] = probs.index(max(probs))\n",
    "        \n",
    "    return labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 算法验证\n",
    "\n",
    "我们人为生成300个分别来自3个不同高斯分布的数据样本，组成验证数据集。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 生成来自高斯混合分布的数据\n",
    "# 数据集来自3个高斯分布，每个分布产生100个数据样本，数据集的维度为2\n",
    "#np.random.seed(5)\n",
    "\n",
    "mus = [[0, 4], [-2, 0], [4, 4]]\n",
    "\n",
    "sigs = [[[3, 0], [0, 0.5]], [[1, 0], [0, 2]], [[1, 0], [0,1]]]\n",
    "\n",
    "\n",
    "# 数据集X\n",
    "multi_data = [np.random.multivariate_normal(mu, sig, 100) for mu, sig in zip(mus, sigs)]\n",
    "\n",
    "data = np.vstack(multi_data)\n",
    "\n",
    "## 数据集的类别标签为0，1，2\n",
    "labels = [[value]*100 for value in range(3)]\n",
    "## 把嵌套数组labels拉平\n",
    "multi_labels = [y for x in labels for y in x]\n",
    "\n",
    "\n",
    "# 数据集标签合并\n",
    "final_data = []\n",
    "\n",
    "for v1, v2 in zip(data, multi_labels):\n",
    "    list_v1 = v1.tolist()\n",
    "    list_v1.append(v2)\n",
    "    final_data.append(list_v1) \n",
    "\n",
    "##np.random.shuffle(final_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "把数据集转化为`Pandas`中的`DataFrame`格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-3.863865</td>\n",
       "      <td>5.011869</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.742150</td>\n",
       "      <td>3.624853</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.435850</td>\n",
       "      <td>5.268131</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.613564</td>\n",
       "      <td>3.530604</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.916746</td>\n",
       "      <td>4.839165</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         x1        x2  label\n",
       "0 -3.863865  5.011869      0\n",
       "1  1.742150  3.624853      0\n",
       "2  0.435850  5.268131      0\n",
       "3  3.613564  3.530604      0\n",
       "4  0.916746  4.839165      0"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_data = pd.DataFrame(final_data, columns = ['x1', 'x2', 'label'])\n",
    "final_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试数据集准备好之后，我们把`final_data`代入函数中进行计算。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reach stopping criterion!\n"
     ]
    }
   ],
   "source": [
    "mus, sigmas = train_GMM(final_data, 3, 1000, 1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.091542  ,  3.98930488],\n",
       "       [-2.07339681, -0.05747473],\n",
       "       [ 3.90201609,  4.13091042]])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 根据训练集生成的均值\n",
    "mus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 3.05429524, -0.14215967],\n",
       "        [-0.14215967,  0.52348837]],\n",
       "\n",
       "       [[ 1.11471389,  0.08816557],\n",
       "        [ 0.08816557,  1.88432252]],\n",
       "\n",
       "       [[ 1.20274147, -0.02181217],\n",
       "        [-0.02181217,  1.09511199]]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 根据训练集生成的协方差\n",
    "sigmas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  2.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  2.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  2.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  2.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  2.  0.  0.  0.  0.  0.  2.  2.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  2.  0.  0.  0.  0.  0.  0.  0.  0.  2.  0.  0.  0.  0.\n",
      "  2.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  0.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  0.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  2.  2.  0.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.]\n"
     ]
    }
   ],
   "source": [
    "# 在训练集上预测的标签\n",
    "\n",
    "label_pred = predict(final_data, mus, sigmas)\n",
    "print label_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9567\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      0.90      0.93       100\n",
      "          1       1.00      0.98      0.99       100\n",
      "          2       0.91      0.99      0.95       100\n",
      "\n",
      "avg / total       0.96      0.96      0.96       300\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD3CAYAAABYUUzPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE11JREFUeJzt3XuYVXW9x/H33gMojIgePWoJeCH9dZRLHC0VRTiIt7Jj\nejrWSUvzlgkdKcpQQVPxgiIaalEk3tLnPGppZZl2EbyGeBtB4JfklYw8aSrMIDDOPn/MSBxL9qB7\n7bX8zfvls59n9t7Mb77DI5/5znf91lqlSqWCJKn2ynkXIEmpMmAlKSMGrCRlxICVpIwYsJKUkW5Z\nLn7gwCPcopCxnz3w3bxLSN7KZS/mXUKX0GfnQaX3usbg7UZ0OnOeeG7Oe/561djBSlJGMu1gJame\nSqXMm9INYsBKSkapVKxfyg1YSckoYwcrSZlwRCBJGSk7IpCkbBStgy1W3EtSQuxgJSWjodSQdwn/\njwErKRlFGxEYsJKSUS5YwDqDlaSM2MFKSkapYD2jASspGQ1lA1aSMlEq2KmyxYp7SUqIHaykZHiq\nrCRlxH2wkpSRou2DNWAlJaNoB7kMWEnJcAYrSRlxBitJGXEGK0kZcQYrSRlxRCBJGXFEIEkZcUQg\nSRkp2jatYlUjSQmxg5WUDA9ySVJGGgo2IjBgJSXDXQQF0r17N8ZPPplt+m5FS/NKrph8FVQqjD9v\nDFQqPLvkBa6YfBWVSiXvUpPQ1tbG5ClTiU89RY/uPTh74mn079c377KSsSD+niuu+SEzLjiHF178\nE+dcdiWUYMB2/Tn1pOMpF+x2Kl1Bl/4bP/jT+7Gy5Q3GHTmR75w/izFnHMuJpx7NtZf/D+OPPotS\nqcReo3bPu8xk/Hb2PaxatZobZs1k3Ngvc/Fl0/MuKRnX/eg2zrt8BqvXrAHgsquu5aTPf5aZUyZT\nqVSYM3dezhXWR6lU6vSjHrp0wPYf0Jd59z0GwNJn/0T/Hfuy0y478sS8hQDMu/cxhu45OM8Sk/Jo\nUxP7DNsDgCGDBrJw0eKcK0pH3222Ycrp31j7fPGSp/nXgbsCMGy3ocx7/Im8SqurcqnU6Udd6qnL\nVymoPyx+lj1G7AbAhwfvxBZb/ROl8t/+4luaV9LYu1de5SWnubmFTRo3Wfu8XG6gtbU1x4rSMWrv\nPenW0LD2eYXK2i6tV8+erGhpyau0uiptwH/10KUD9s5b76ZlxUouue4c9t7vYyxZ+DRtb7atfb9X\nY0+aX2/OscK0NDb2onmdf+htlTa6devShwEys26H1rJyJb0bG3Ospn6K1sGu9//uEMLdwEZve7kE\nVGKMwzKrqk7CwAE8Pnc+37voWnbadUe2+uCW/PXl1xj80V14Yt5CPjp8KE0PLci7zGQMHTKY2ffc\nz0H770fT/AXsNGBA3iUla+cdd+CR+QvYbdBAHnjkMXYfPDDvkuri/bYPdgIwEzgMSO53uT8+t4yj\nx36Gz55wOM3Lm5l25gx69tqYcd/6Et26d+OFZ/7IvXf9Lu8yk7HfyBE8OHceRx17IhUqnHvmGXmX\nlKxTjjua8y+fwZrWG9mh37aMGrZn3iXVRa060xBCd+BaYHvgTeAE2jPwGqACLADGxBjb3mEJAErV\ntiCFEL4BLIkx3rqhRR448Aj3N2XsZw98N+8Skrdy2Yt5l9Al9Nl50HtOx+OGjel05lz1wJXv+PVC\nCIcCR8YYjwgh7A+cBHQHpsUYZ4cQZgB3VsvFqgOwGOPFnS1YkvJUw9nq74FuIYQysCmwBtgTmNPx\n/h3AAcB7C1hJer+o4Qx2Be3jgcXAlsAhwL4xxrc65OVAn2qLdOldBJLSUsNdBF+lfQSwMzCE9nls\nj3Xe7w28WrWed/uNSFLR1PBMrr8Cr3V8/Art89fHQggjO147GLi32iKOCCQlo4YnEFwKzAoh3Et7\n53o68DAwM4TQA1gE3FJtEQNWkt4mxrgCOOIfvDViQ9YxYCUlo1ys8wwMWEnpaCjYJRkNWEnJKNqp\nssWKe0lKiB2spGSU63QZws4yYCUlo2gjAgNWUjK86aEkZaRg+WrASkqHHawkZaRe99rqLANWUjI8\nyCVJGXFEIEkZKVi+GrCS0mEHK0kZ8SCXJGXEDlaSMlKwfPVqWpKUFTtYScnwgtuSlJGijQgMWEnJ\nKNpBrmL105KUEDtYSclwH6wkZcSLvUhSRhrKxQpYZ7CSlJFMO9ifzJme5fICPjbkP/MuIXkPNd2c\ndwnqJEcEkpSRgk0IDFhJ6bCDlaSMFCxfDVhJ6SjamVwGrKRkeKKBJGWkYA2sASspHUUbEXiigSRl\nxA5WUjLKBdsIa8BKSkYt98GGEE4D/h3oAXwHmANcA1SABcCYGGPb+tZwRCApGeVS5x/rE0IYCQwD\n9gZGAP2AacDEGONwoAQcWrWe9/j9SFKKDgTmA7cCPwNuB3ajvYsFuAMYXW0RRwSSklHDEcGWwHbA\nIcAOwE+Bcoyx0vH+cqBPtUUMWEnJqOH1YF8GFscYVwMxhPAG7WOCt/QGXq22iCMCSckolUqdflRx\nH3BQCKEUQvgg0Aj8pmM2C3AwcG+1RexgJeltYoy3hxD2BR6ivREdAzwDzAwh9AAWAbdUW8eAlZSM\nWp7IFWM89R+8PGJD1jBgJSXD68FKUkYKlq8GrKR0FO1iLwaspGQULF8NWEnpcAYrSRkpWL4asJLS\nUbQO1jO5JCkjdrCSklHDaxHUhAErKRkFmxAYsJLS4QxWkroIO1hJyShYA2vASkqHd5UtoDWtrZx1\n3oW8uGwZq1ev4cRjvsDI4XvnXVYSuvfozjkXncq2/T9A84oWLjjz2/TsuTFnnPdV3mx9k+eeWcrZ\nE6ZSqVSqL6aq2tramDxlKvGpp+jRvQdnTzyN/v365l1W3RRtBmvAAj//5V1s1qcP5581kddef50j\njj7OgK2Rwz/7CVpaVvKFw8ey3Y79mHD2f/PGylV8f/r13Dd7LudfejrDR+3JPb95MO9Sk/Db2few\natVqbpg1k6b5C7j4sulcfslFeZfVZW3wQa4QwkZZFJKnA0aNZMwJxwFQqVRoaGjIt6CEDPjQdtw3\n+yEAnnv6BXYY0J/FTz7Fppv1BqDXJr1oXdOaZ4lJebSpiX2G7QHAkEEDWbhocc4V1Vep1PlHPbxj\nwIYQPhlCeC6EsCSE8Jl13rqjDnXVVa9evWhs7EVzcwvjzziTsScel3dJyYiLlrDvqD0BGPSRf2Gr\nbbZk6fMv8s2zxnLrr69hiy035+HfPZ5zlelobm5hk8ZN1j4vlxtobe06P8BqeE+umlhfB3sG8BFg\nD+BLIYSjO14v1pCjRpb9+SWO/8o4DjnoAD5+wP55l5OM2266g+YVLVx907cZdeA+LFrwFF+feDLH\nHjGOw0Yfw+0/vovxE7+cd5nJaGzsRXNLy9rnbZU2unXrOpPA900HC6yOMf41xvgycCgwNoTwb0By\nRyNefuUVTho3nnEnf4nDDvlE3uUkZdfBH2buA4/yxSNO4Ve/mMPS51/ktdeWs2JFMwAv/fllNt20\nd85VpmPokMHce3/7PLtp/gJ2GjAg54rqq6Fc6vSjHtb3o+3ZEMI0YFKMcXkI4XDgTmCzulRWRz+4\n9oe8vnwF37/6Or5/9XUAXDntIjbeKLlxc909/+xSLvzaJI4fcyTLX1/B2d+cyrb9P8CU6ZNoffNN\nWtes4ZwJl+RdZjL2GzmCB+fO46hjT6RChXPPPCPvkrq00jttjwkhdAOOAm6KMbZ0vLY1cFqMcVxn\nFn/j5WXJdbtFs+fun8u7hOQ91HRz3iV0CT023eI9t5W/njCj05kz+sKTMm9j37GDjTG2Ate87bU/\nA50KV0mqt4Jtg3UfrKR0lDyTS5KyYQcrSRnxVFlJykjB8tWAlZQOO1hJykjB8tU7GkhSVuxgJSWj\nVC5Wz2jASkpG0UYEBqykZBTtRINi9dOSlBA7WEnJcEQgSRlxH6wkZaTWt+0OIWwFPALsD7x1hcEK\nsAAYE2NsW289Na1GkhIRQugOfA9Y2fHSNGBijHE47bfOOrTaGgaspGTU+J5cU4EZwIsdz3cD5nR8\nfAcwutoCBqykZNTqrrIhhGOA/40x3rnu8jHGt+6YsBzoU60eZ7CS0lG7lvFYoBJCGE373bWvA7Za\n5/3ewKvVFjFgJSWjVrsIYoz7vvVxCGE2cBJwcQhhZIxxNnAwcHe1dQxYScnIeJfWeGBmCKEHsAi4\npdonGLCSkpHFPtgY48h1no7YkM81YCUlo2DnGRiwkhJSsIR1m5YkZcQOVlIyyg3F6mANWEnJ8GIv\nkpSRguWrM1hJyoodrKR0FKyFNWAlJaNo9+QyYCUlo2gB6wxWkjKSaQdbajC/s/ZQ0815l5C83Qcd\nnncJXcITz82p/oeqKNgI1hGBpHQUbURgwEpKhicaSFJWipWvBqykdNjBSlJGDFhJykrBNi4ZsJKS\nUbQOtmB5L0npsIOVlAz3wUpSRgxYScqKM1hJ6hrsYCUlo2ANrAErKR1F26ZlwEpKRtEukVqsaiQp\nIXawktJRrAmBASspHc5gJSkjnmggSRkplYt1WKlY1UhSQuxgJaWjWBMCA1ZSOpzBSlJW3EUgSdmo\n1TatEEJ3YBawPbARMBlYCFwDVIAFwJgYY9v61vEgl6R0lEudf6zfUcDLMcbhwEHAFcA0YGLHayXg\n0KrlvMdvR5IKo1QqdfpRxc3ApLeWBVqB3YA5Ha/dAYyutogjAknpqNEINsa4AiCE0Bu4BZgITI0x\nVjr+yHKgT7V17GAlJaOGHSwhhH7A3cD1McYbgXXnrb2BV6utYcBK0tuEELYG7gK+GWOc1fHyYyGE\nkR0fHwzcW20dRwSS0lG7fbCnA5sDk0IIb81iTwGmhxB6AItoHx2slwG7jicWPMllV85g1ncvz7uU\nJLW1tTF5ylTiU0/Ro3sPzp54Gv379c27rPe97j26c+7UCfTt9wFWrGjh/EmX0rhJLyaeN541q9ew\neOESpnxrOpVKpfpi73O1uhZBjPEU2gP17UZsyDoGbIdZ19/A7XfcRc+NN867lGT9dvY9rFq1mhtm\nzaRp/gIuvmw6l19yUd5lve/9x38dQkvzSo467GS237Efp50zjs0235QLvzWdpkeeZOzXj+PjnxrN\nz2/9Vd6lZq5olyvcoLgPIfQMIWyUVTF56rfttlx64eS8y0jao01N7DNsDwCGDBrIwkWLc64oDQN2\n2p77Zs8F4NmnX2DHD23H1tv8M02PPAnA4w8vYOjug/Issctab8CGEHYJIdwWQrg6hDCa9rnDwhDC\nIfUpr372HzWSbt1s6LPU3NzCJo2brH1eLjfQ2tqaY0VpWPzkEvbdby8ABg/dha222ZI/Ll3GbnsM\nAWDE6GH07NUzzxLrp3YnGtSmnCrvzwAuBWbTPtD9GDAUOC3bspSixsZeNLe0rH3eVmnzh1oN3HbT\nL2he3sw1t1zOqAOHs3D+75k0/gKOP/lIZt44jVf+8ldefaXqjqIk1HKbVi1UC9hyjHFOjPFa4LYY\n40sxxtdpP6tB2iBDhwzm3vsfBKBp/gJ2GjAg54rSsOuQDzP3/kc55tNf4a6fz2bp8y8yfNReTDhl\nMid87mv02bwPD973cN5l1kWpodzpRz1Uax9iCOEHwIkxxmMAQggTgGVZF6b07DdyBA/OncdRx55I\nhQrnnnlG3iUl4flnljL2ijM54SufZ/nrKzjrG1PYZVBg5o3TeOONVcx78DHuu3tu3mV2SaX1bd0I\nIZSBT8YYf7LOa0cBP44xtrzjJ3ZY9epL6e8LyVmp3JB3CcnbfdDheZfQJTzx3Jz3/Hv7Xx5+sNOZ\ns+Xue2U+J1hvB9txKa6fvO21H2ZakSS9S0XbpuURBknpMGAlKRveMkaSsmIHK0kZMWAlKRse5JKk\nrBRsBusFtyUpI3awkpJRKhWrZzRgJSWjVhfcrhUDVlI6nMFKUtdgByspGW7TkqSsGLCSlI1SQ7Eu\n3+kMVpIyYgcrKR2OCCQpGx7kkqSseCaXJGXDC25LUlYcEUhSNpzBSlJWnMFKUkYKNoMtVtxLUkLs\nYCUlwxmsJGWkVC7WtQgMWEnpKNhBrmJVI0kJsYOVlIxanckVQigD3wGGAKuA42OMSzZ0HTtYSeko\nlTr/WL9PARvHGPcCJgCXvJtyDFhJySiVGzr9qGIf4JcAMcbfAbu/m3oyHRFstNlWxdozIb0LTzw3\nJ+8S1Ek9Nt2iVpmzKfDaOs/fDCF0izG2bsgidrCS9PdeB3qv87y8oeEKBqwk/SP3Ax8HCCHsCcx/\nN4u4i0CS/t6twP4hhAeAEvDFd7NIqVKp1LQqSVI7RwSSlBEDVpIyYsBKUkY8yLWOEMIewJQY48i8\na0lRCKE7MAvYHtgImBxj/GmuRSUmhNAAzAQCUAFOijEuyLeqrssOtkMI4VTgB8DGedeSsKOAl2OM\nw4GDgCtyridFnwSIMe4NTATOy7ecrs2A/Zs/AIfnXUTibgYmdXxcAjZ447bWL8Z4G3Bix9PtgFdz\nLKfLc0TQIcb4oxDC9nnXkbIY4wqAEEJv4BbaOyzVWIyxNYRwLXAY8Om86+nK7GBVVyGEfsDdwPUx\nxhvzridVMcajgZ2BmSGExrzr6arsYFU3IYStgbuAsTHG3+RdT4pCCJ8H+sYYLwBagLaOh3JgwKqe\nTgc2ByaFEN6axR4cY1yZY02p+TFwdQjhHqA7MM6/3/x4qqwkZcQZrCRlxICVpIwYsJKUEQNWkjJi\nwEpSRgxYScqIAStJGfk/9ERbhzTzgEIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x111cc8810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## 模型评价\n",
    "evaluate(label_pred, final_data['label'].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 使用`sklearn`的`Gaussian`函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.mixture import GaussianMixture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9567\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.92      0.93       100\n",
      "          1       1.00      0.97      0.98       100\n",
      "          2       0.92      0.98      0.95       100\n",
      "\n",
      "avg / total       0.96      0.96      0.96       300\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD3CAYAAABYUUzPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE6dJREFUeJzt3XuYVXW9x/H33sMMd8n0qBmIivbzlEoefTJJgeNd0zya\nWafQzNJMTK2OCjlgKqYiXjGPheGlrPOolXkq05MJXjK0KISEX1peM4lIBWYIHPY+f8xAZDZ7kL32\nWv7m/fJZzzN7b/j5hQc+fOe7fmutUrVaRZJUf+W8C5CkVBmwkpQRA1aSMmLASlJGDFhJykifLBcf\nvdMRblHI2E8emZl3CclbtXRJ3iX0CoO326m0sWvsOnxMjzPnsWdmb/T/rxY7WEnKSKYdrCQ1UqmU\neVO6QQxYSckolYr1TbkBKykZZexgJSkTjggkKSNlRwSSlI2idbDFintJSogdrKRkNJWa8i7h7xiw\nkpJRtBGBASspGeWCBawzWEnKiB2spGSUCtYzGrCSktFUNmAlKROlgl0qW6y4l6SE2MFKSoaXykpS\nRtwHK0kZKdo+WANWUjKKdpLLgJWUDGewkpQRZ7CSlBFnsJKUEWewkpQRRwSSlBFHBJKUEUcEkpSR\nom3TKlY1kpQQO1hJyfAklyRlpKlgIwIDVlIy3EVQIM3NfZhw0WlsPXQr2tvaueL8r9Kvfz9Obz2R\nSqXCq6s7uPDsK3hp6St5l5qESqXClEumEZ94gpbmFs5rncg2w4bmXVZSOjo6OHfaVfxx8WLK5SZa\nzxjPtv4e56ZXB+zhxxzIyva/8pmPnMWw7d7OGZM+TUvfZq6aMoMnFz3FBz58EB898YN85eKZeZea\nhJ/Oup9Vq1Zzy8wZzJu/gEuvvJrpl03Nu6ykPPjoL1mzZg0zr5jKz+f+mq/c+E0unTQh77Iaxhls\ngQwfMYw5988F4Lmn/sDw7Ydy8ofPZOmSlwBoampi9arVeZaYlLnz5rH3qD0BGLnLzjy+cFHOFaVn\n+Nu3Zs2aNVQqFdra2+nTpynvkhqqaCOCYk2EG+zJRU8xauweALxz5DvYfMu3rhsH7LzbThz1sUO5\n9cY78ywxKW1t7QwaOGjd63K5iY6OjhwrSk///v14YfGfOPrE8Vx45Vf4yBGH5V1SQ5U24L9G6NUd\n7I++8xOGbz+Ma265iAVzF/Lb3/yOSqXCvofszbEnf4izPn0Br7y0LO8ykzFw4ADa2tvXva5UK/Tp\n06v/CNbdt757J3vtvhunnnAcLy5ZwmfOnsT/XHc1fVta8i6tIYrWwXb7pzuEcB/Q9zVvl4BqjHFU\nZlU1yE677Mgvfz6Pay7+OmHnHdhy6y044PAxfODDB3Haceew/JUVeZeYlN1G7sqs+x/i4AP2Y978\nBew4YkTeJSVnk0GD1o0FhgweTEdH57igt6jXDDaE0AzcBGwLrAFOBDqAG4EqsAAYH2Ps9je3Vvsw\nAZgBHNm1eFKef/oFzr38TI49+UOsWNbG1NZruPHOq1n8xyVMmd55YuDXj/6GG6Z/O+dK07Df2DE8\nPOdRxp1wElWqXDD5nLxLSs5Hj/oA518+nU99YSKvdrzK+E+Mo3+/fnmX1TB17GAPBfrEGEeFEA4A\nLgSagdYY46wQwnXAEcD3uluk24CNMc4JIXwD2DXG2O1Cb0avvLycz58w+e/eO+y943KqJn3lcpnJ\nE8/Ku4ykDejfn4vP6b2/x3Wcrf4W6BNCKAObAK8C7wVmd31+F3AgGxOwADHGSzeuTklqjDp2sCvo\nHA8sAjYHDgNGxxirXZ8vB4bUrKde1UhS3kqlUo+PGj4H3B1jfAcwks557PpnCgcDL9daxICVlIxy\nqdTjo4aXgLWXcP6Fzvnrr0IIY7veOwR4oNYi7pGRlIw6Xsl1BTAzhPAAnZ3rF4FfADNCCC3AQuD2\nWosYsJKSUa+TXDHGFcAxr/PRmA1ZxxGBJGXEDlZSMsrFupDLgJWUjqZysb4pN2AlJaNotyssVtxL\nUkLsYCUlo9yg2xD2lAErKRlFGxEYsJKS8aa6H6wkvZkULF8NWEnpsIOVpIw06llbPWXASkqGJ7kk\nKSOOCCQpIwXLVwNWUjrsYCUpI57kkqSM2MFKUkYKlq/eTUuSsmIHKykZ3nBbkjJStBGBASspGUU7\nyVWsflqSEmIHKykZ7oOVpIx4sxdJykhTuVgB6wxWkjKSaQd79wPXZrm8gPeM/FDeJSTvkXm35V2C\nesgRgSRlpGATAgNWUjrsYCUpIwXLVwNWUjqKdiWXASspGV5oIEkZKVgDa8BKSkfRRgReaCBJGbGD\nlZSMch03woYQJgIfAFqAa4HZwI1AFVgAjI8xVrqtp27VSFLOSqVSj4/uhBDGAqOA9wFjgGHA5UBr\njHEfoAQcUaseA1ZSMsqlnh81HATMB74H/C/wA2B3OrtYgLuA/Wst4ohAkv7R5sBw4DBgO+BOoBxj\nrHZ9vhwYUmsRA1ZSMup4qexSYFGMcTUQQwh/pXNMsNZg4OVaizgikJSMpnKpx0cNDwIHhxBKIYSt\ngYHAvV2zWYBDgAdqLWIHKykZ9epgY4w/CCGMBh6hsxEdDzwFzAghtAALgdtrrWPAStLriDGe9Tpv\nj9mQNQxYScko2IVcBqykdHg/WEnKSMHy1YCVlI6i3ezFgJWUjILlqwErKR3OYCUpIwXLVwNWUjqK\n1sF6qawkZcQOVlIyenCPgYYyYCUlo2ATAgNWUjqcwUpSL2EHKykZBWtgDVhJ6ajnU2XrwYAF1qxZ\nw/mXXMbTzz1HiRKtZ36OHbbfLu+yktDc0sz5U8/i7du8jbYV7Vw0+SpO+fwn2Pxf3grA1kO34rFf\nPc6E06bkXGkaKpUKUy6ZRnziCVqaWzivdSLbDBuad1kNU7QZrAELzH7oYQBu+u/pPDr311zzta9z\n5cX+ha+Hoz7yftrbV3LcUacyfPthTDjvNE75+NkADN5kENd/+3KmXXBtzlWm46ez7mfVqtXcMnMG\n8+Yv4NIrr2b6ZVPzLqvX2uCADSH0jTGuyqKYvOw7em9Gj9oLgD8uXsygQYNyrigdI3YYzoOzHgHg\nmd8/x3Yjtln32Wc+dzzfvul7/HnJX/IqLzlz581j71F7AjByl515fOGinCtqrII1sP98F0EI4fAQ\nwjMhhCdDCB9e76O7GlBXw/Xp00TrlIu55IrpHHrgfnmXk4y48ElG7/teAHZ597+yxVabUy6X2XSz\nt7DnqH/jztvvzrnCtLS1tTNo4N8ahHK5iY6OjhwraqxSqdTjoxG626Z1DvBuYE/g0yGEj3e9X7B/\nI+pnSusEvv/tm7ngkstYuXJl3uUk4Y5b76JtRTs33HoV+x60NwsXPEGlUuGAQ0Zz1533UqlU8i4x\nKQMHDqCtvX3d60q1Qp8+vWcSWCr1/GiE7gJ2dYzxpRjjUuAI4NQQwr8D1caU1jg/+PE9fP0b3wKg\nX7++lMplSmW3CNfDu3bdiTk/m8snjjmd//vRbJ5/9gUA9nzf7utGB6qf3UbuygNd5xTmzV/AjiNG\n5FxRY9Xxsd110d0/bU+HEC4HJsUYl4cQjgLuBt7SkMoaaL8x+zD5y1M5YfzpdHSs4czTxtOvb9+8\ny0rCs08/z8Wfn8Snxn+M5ctWcN7Z0wDYdvth/KErbFU/+40dw8NzHmXcCSdRpcoFk8/Ju6RerVSt\nvn5DGkLoA4wDbo0xtne9tyUwMcZ4Rk8WX7nkD8l1u0Wz13uOzbuE5D0y77a8S+gVWjbZbKPbyp9M\nuK7HmbP/xSdn3sb+0w42xtgB3Pia9xYDPQpXSWq0ou0i6D3Tb0nJK3kllyRlww5WkjLipbKSlJGC\n5asBKykddrCSlJGC5atPNJCkrNjBSkpG0S5xN2AlJaNoIwIDVlIyinahQbH6aUlKiB2spGQ4IpCk\njNR7H2wIYQvgl8ABwNobYFWBBcD4GGO3d4x3RCApGeVyqcdHLSGEZuCrwNrHm1wOtMYY96HzyS5H\n1KxnI34tkpSyacB1wNo7w+8OzO76+i5g/1oLGLCSklGvZ3KFEI4HlsQY138qZynGuPaG3suBIbXq\ncQYrKRl1nMGeAFRDCPvT+fDXm4Et1vt8MPByrUUMWEnpqNP35DHG0Wu/DiHMAk4GLg0hjI0xzgIO\nAe6rtY4BKykZGd9N6wvAjBBCC7AQuL3WTzBgJSUji3yNMY5d7+WYDfm5BqykZHg/WEnKSMHy1YCV\nlJCCJaz7YCUpI3awkpJRbipWB2vASkqGJ7kkKSMFy1dnsJKUFTtYSekoWAtrwEpKRtGeyWXASkpG\n0QLWGawkZSTTDrZUbspyeQGPzLst7xKSt8cuR+VdQq/w2DOza/+gGgo2gnVEICkdRRsRGLCSkuGF\nBpKUlWLlqwErKR12sJKUEQNWkrJSsI2nBqykZBStgy1Y3ktSOuxgJSXDfbCSlBEDVpKy4gxWknoH\nO1hJyShYA2vASkpH0bZpGbCSklFqKtbUs1jVSFJC7GAlpaNYEwIDVlI6nMFKUka80ECSMlIqF+u0\nUrGqkaSE2MFKSkexJgQGrKR0OIOVpKy4i0CSslGvbVohhGZgJrAt0BeYAjwO3AhUgQXA+Bhjpbt1\nPMklKR3lUs+P7o0DlsYY9wEOBq4BLgdau94rAUfULGcjfzmSVBilUqnHRw23AZPWLgt0ALsDs7ve\nuwvYv9YijggkpaNOI9gY4wqAEMJg4HagFZgWY6x2/ZDlwJBa69jBSkpGHTtYQgjDgPuAb8QYvwWs\nP28dDLxcaw0DVpJeI4SwJXAPcHaMcWbX278KIYzt+voQ4IFa6zgikJSO+u2D/SKwKTAphLB2Fns6\ncHUIoQVYSOfooFsGLPBqRwfnXngxL7z4IqtXv8pJxx/H2H3el3dZyalUKky5ZBrxiSdoaW7hvNaJ\nbDNsaN5lvek1tzRzwbQJDB32NlasaOfLk66g/4D+TLrw83SsWcMzTz3Pl86aSrVarb3Ym1y97kUQ\nYzydzkB9rTEbso4BC/zwx/fwliFD+PK5rbyybBnHfPyTBmwGfjrrflatWs0tM2cwb/4CLr3yaqZf\nNjXvst70Pvifh9HetpJxR57CttsPY+L5Z/DXlau47uqbePC+OVx0VSuj992L2ff+LO9SM1e02xVu\nUNyHEPqHEPpmVUxeDtx3LONP/CQA1WqVpqamfAtK1Nx589h71J4AjNxlZx5fuCjnitIwYsdteXDW\nHACe/v1zbL/DcBb95gmGvGUTAAYOHEBHR0eeJfZa3QZsCOGdIYQ7Qgg3hBD2p3Pu8HgI4bDGlNcY\nAwYMYODAAbS1tfOFcyZz6kmfzLukJLW1tTNo4KB1r8vlJv/i18Gi3zzJ6P32AmDX3d7JFlttznPP\n/oEJXzqN7997M5ttvimP/vzXOVfZIPW70KA+5dT4/DrgCmAWnQPd9wC7AROzLavxXlz8Jz712TM4\n7OADOfTAA/IuJ0kDBw6grb193etKtUKfPk6pNtYdt/6ItuVt3Hj7dPY9aB8en/9bzpp0Kscf/VmO\n2O847vzu3fxX6yl5l9kQ9dymVQ+1ArYcY5wdY7wJuCPG+KcY4zI6r2pIxtK//IWTz/gCZ5zyaY48\n7P15l5Os3UbuygMPPQzAvPkL2HHEiJwrSsO7Ru7EnIfmcvzRn+WeH87i+Wdf4JWXl7FiRRsASxYv\nZZMhg3OusjFKTeUeH41Qq32IIYTrgZNijMcDhBAmAC9mXVgjXX/TN1m2fAVfu+FmvnbDzQB85fKp\n9Oub3Lg5V/uNHcPDcx5l3AknUaXKBZPPybukJDz71POces1kTvzssSxftoJzz7yEodtszdRrzmVN\nxxpefbWD8yZcmneZvVKpu60bIYQycHiM8fvrvTcO+G6Msf2f/sQuf136Yvr7QnJWbm7Ou4Tk7bHL\nUXmX0Cs89szsjf6+/c+/eLjHmbP5HntlPifotoPtuhXX91/z3jczrUiS3qCibdPyDIOkdBiwkpQN\nHxkjSVmxg5WkjBiwkpQNT3JJUlYKNoP1htuSlBE7WEnJKJWK1TMasJKSUa8bbteLASspHc5gJal3\nsIOVlAy3aUlSVgxYScpGqWDP03MGK0kZsYOVlA5HBJKUDU9ySVJWvJJLkrLhDbclKSuOCCQpG85g\nJSkrzmAlKSMFm8EWK+4lKSF2sJKS4QxWkjJSKhfrXgQGrKR0FOwkV7GqkaSE2MFKSka9ruQKIZSB\na4GRwCrgUzHGJzd0HTtYSekolXp+dO8/gH4xxr2ACcBlb6QcA1ZSMkrlph4fNewN/BggxvhzYI83\nUk+mI4J+m21VrD0T0hvw2DOz8y5BPdSyyWb1ypxNgFfWe70mhNAnxtixIYvYwUrSP1oGDF7vdXlD\nwxUMWEl6PQ8BhwKEEN4LzH8ji7iLQJL+0feAA0IIPwNKwCfeyCKlarVa16okSZ0cEUhSRgxYScqI\nAStJGfEk13pCCHsCl8QYx+ZdS4pCCM3ATGBboC8wJcZ4Z65FJSaE0ATMAAJQBU6OMS7It6reyw62\nSwjhLOB6oF/etSRsHLA0xrgPcDBwTc71pOhwgBjj+4BW4MJ8y+ndDNi/+R1wVN5FJO42YFLX1yVg\ngzduq3sxxjuAk7peDgdezrGcXs8RQZcY43dCCNvmXUfKYowrAEIIg4Hb6eywVGcxxo4Qwk3AkcDR\nedfTm9nBqqFCCMOA+4BvxBi/lXc9qYoxfhx4BzAjhDAw73p6KztYNUwIYUvgHuDUGOO9edeTohDC\nscDQGONFQDtQ6TqUAwNWjfRFYFNgUghh7Sz2kBjjyhxrSs13gRtCCPcDzcAZ/v7mx0tlJSkjzmAl\nKSMGrCRlxICVpIwYsJKUEQNWkjJiwEpSRgxYScrI/wP86HQxl7G4QwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11814d5d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gmm_model = GaussianMixture(n_components = 3, covariance_type = 'diag', init_params='random', max_iter=200)\n",
    "\n",
    "gmm_model.means_init = np.array([final_data[final_data['label'] == num].mean(axis = 0)[:-1] for num in range(3)])\n",
    "\n",
    "gmm_model.fit(final_data.iloc[:,:-1])\n",
    "\n",
    "y_pred = gmm_model.predict(final_data.iloc[:,:-1])\n",
    "\n",
    "\n",
    "\n",
    "diff = y_pred  - final_data['label'].values\n",
    "\n",
    "evaluate(y_pred, final_data['label'].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来，我们查看训练得到的均值和协方差矩阵。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.14315648,  3.99572962],\n",
       "       [-2.08035331, -0.11343775],\n",
       "       [ 3.8482867 ,  4.0898108 ]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmm_model.means_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 4.01553565,  0.61132795],\n",
       "       [ 1.0545162 ,  1.44822495],\n",
       "       [ 1.01247237,  1.14685908]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmm_model.covariances_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 真实数据集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用`sklearn`中的`GaussianMixture`模型，查看数据集是否适合使用高斯混合模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8762\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.84      0.77      0.81        70\n",
      "          1       0.97      0.86      0.91        70\n",
      "          2       0.83      1.00      0.91        70\n",
      "\n",
      "avg / total       0.88      0.88      0.87       210\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD3CAYAAABYUUzPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEppJREFUeJzt3XuYVXW9x/H32gMDglCmxfEKhfFLT4iKR0hJ8FInfTLS\nzDrnwTStORhoqEfzAl5OmoJKGd4SNY3UQ5moWaaViuYNlbyQ8gtNMRVU6KjIKDDOPn/MSGg5M8is\nWYvffr981vPMWtu9/Lqfhw/f+a7fWjurVqtIkjpfpegCJClVBqwk5cSAlaScGLCSlBMDVpJy0i3P\nk0/Y4xiXKOTszJlHF11C8l6ZN7/oEmrCprvvma3rObbrP7LDmfPowtnr/N9rjx2sJOUk1w5WkrpS\nluXelK4VA1ZSMrKsXL+UG7CSklHBDlaScuGIQJJyUnFEIEn5KFsHW664l6SE2MFKSkZdVld0Ce9g\nwEpKRtlGBAaspGRUShawzmAlKSd2sJKSkZWsZzRgJSWjrmLASlIuspLdKluuuJekhNjBSkqGt8pK\nUk5cBytJOSnbOlgDVlIyynaRy4CVlAxnsJKUE2ewkpQTZ7CSlBNnsJKUE0cEkpQTRwSSlBNHBJKU\nE5dpSdJ6IIRwAvAFoB64EJgNXAFUgXnAuBhjc1vnKFfcS9I6yLKsw1tbQgijgF2AXYGRwJbAVGBi\njPHTQAaMbq8eA1ZSMuqySoe3dvw78BgwC/glcBMwlJYuFuBmYK/2TuKIQFIyOnEVwSZAf+DzwEeB\nG4FKjLHa+voy4APtnaTmA/aYHx3FiuVvArB08d+4ZspMAHbcYwd2228EPzhiWpHlJWVVUxOnnjmF\nFxa9yMpVK/nmwWMYNWLXostKxuNPP82Prrue8445avWx3815gOtuv4MLv3NsgZWtl5YC82OMK4EY\nQniTljHB2/oAr7R3kpoO2G7du5GRcf7RF73j+OZbb87wfYZBydbUre9+dctv+UDfvpwx6URefe01\nvnJIgwHbSa655VZuvX8OPXvUrz624Nm/8uu776FKtY13pqUTbzT4A/DtEMJUYFOgN/D7EMKoGOMd\nwN7A7e2dpKZnsJsP3Iz6nt0ZO6WBb507lv7bbEWvvr34/GF7M+uC64suLzmf3X0U4755KADVapW6\nurqCK0rHZh/+MN/9r4bV+6++/jrTb7iB8QceUGBVXa+SZR3e2hJjvAn4IzCHlhnsOOAY4LQQwr20\nrCy4tr16arqDXbliJbf97A7u+9X9fHiLTRg7uYHFz7zI9RfdyKoVq4ouLzm9em0AwPLGRv574mmr\nw1brbuSOO7BoyVIA3mpuZsqMnzLugC9R372+nXempTNvNIgxHvdPDo9cm3PUdMC+9NzLLHl+CQAv\nP7eEjfptRLVa5csTvkS3+u78S/9+7DduNLMuuKHgStOx+MWXOPrEkzlwv9Hs89k9iy4nSX9e+CzP\nv/QyU6/+X1Y2rWLhosVM+9nPOeLALxddWu7Wq1tlQwi3Az3edTgDqjHGXXKrqosM33tnNv3oplx7\n3nX03bgvLz+3hMmHnk1zczMf6rcRX5t0kOHaiZb+7W8cfvRxHH/UkQzbaceiy0nWNh8dwBWnTAJg\n0ZKl/M9ll9VEuML697CX44HpwH5AU/7ldK37fj2H//zOVznyvPFUq1WuOXsmzc1t3pihdXDpT67m\ntWXLuOSKGVxyxQwALjj3LHr2ePff4dL7U7YONqtW277CGEI4FngyxjhrbU8+YY9jaufyZUHOnHl0\n0SUk75V584suoSZsuvue65yOh+0yrsOZc9k9F+Sexu3OYGOMZ+ddhCR1hrJ1sDV9kUtSWta3Gawk\nrTfsYCUpJ3awkpSTsn2jQU3fKitJebKDlZSMSrkaWANWUjrqKuX6pdyAlZSMsl3kKlfcS1JC7GAl\nJaNSslUEBqykZJRtRGDASkqGd3JJUk5Klq8GrKR02MFKUk7KdqusASspGV7kkqScOCKQpJyULF8N\nWEnpsIOVpJx4kUuScmIHK0k5KVm++jQtScqLHaykZPjAbUnKSdlGBAaspGSU7SJXufppSUqIHayk\nZLgOVpJy4sNeJCkndZVyBawzWEnKSa4d7KRzDszz9AIO2fvkoktI3ozbphRdgjrIEYEk5aRkEwID\nVlI6OruDDSF8BHgI+AywAXATsKD15YtijDPber8BKykZnZmvIYTuwI+AN1oPDQWmxhjP7eg5DFhJ\nyejkO7nOAS4GTmjdHwqEEMJoWrrYCTHGZW3W05nVSFKRsrX4py0hhEOAl2OMt6xxeA5wbIxxN+Av\nwCnt1WMHKykZndjAHgpUQwh7AdsDPwG+EGNc3Pr6LGBaeycxYCUlo7NGBK1dKgAhhDuAscANIYQj\nYoxzgD1pufjVJgNWkjrmcGBaCGEVsBhoaO8NBqykZFRyWAgbYxy1xu6ua/NeA1ZSMryTS5JyUrY7\nuVymJUk5sYOVlAxHBJKUk7I9D9aAlZSMsnWwzmAlKSd2sJKSUbIG1oCVlI6yjQgMWEnJKFm+GrCS\n0tHJz4NdZwaspGSULF8NWEnpcAYrSTkpWb4asJLSUbYO1hsNJCkndrCSkuGzCCQpJyWbEBiwktLh\nDFaSaoQdrKRklKyBNWAlpSOPb5VdFzUfsH968ikuvHomF5x8Is8tfpHTL76EjIyPbbkFx3z9a1Qq\nTlHW1RcP2Yehu21Pt+513Hrt7Tz+0J/51qmHUq1W+etTz3P55KuoVqtFl5mE5uZmTp98DnHBAuq7\n13PaxBPYasstii6ryziDLZGf3vgrzrzkMlauWgXAD2dcTcOBB3DRqROpVqvc9dDcgitc/207NDBo\nu4GcfNiZnNowhY37fYivHf0VZl40i1O/OZksy9hp5PZFl5mM2+64kxUrVnLV5dOZMP5wzv7BD4su\nqaatdcCGEHrkUUgRNu/3Ec486sjV+/OffoYdtvkEAMO3344HHvtTUaUlY8jwf+XZJ5/nmHPG8Z3v\nH8ncux7hY5/oz+MPRQAevucxBu+8bcFVpmPuI48wYpdhAAwZ/Ekef2J+wRV1rSzr+NYV3nNEEELY\nFzgfWAWcFGOc2frSzcAeXVBb7nYf9m8sevnlvx+oVlf/itGr5wa83vhGQZWlo88H+7DJphszecJ5\nfGTzTThu6pFka8zJ3lj+Jr023KDACtOyfHkjG/becPV+pVJHU1MT3brVxjSwbCOCtj71k4Dtaely\nfx5C6BljvBIo1/9BJ1rzD37jm2/Qp3evAqtJw7JXX+eFZxbxVtNbLFr4IqtWrGLjfhutfn2D3j1Z\n/npjgRWmpXfvXixv/Pvn2VxtrplwhfKtImhrRLAyxvh/McalwGhgfAhhdyDZqxGDBvRn7uNPAHDf\nw48y5BODCq5o/RcfXsCQXT4JwEabfJAeG9Qzb84TbDs0ALD9LoOZ/8cFRZaYlB2GbMddd98LwCOP\nzePjAwcWXFHXqqtkHd66Qlt/tT0TQpgKTIoxLgsh7A/cAnywSyorwBFj/oOzLrmcVW81MWCzzdh9\n2M5Fl7Tem/uHR9lmx0F878qJZJWMyydfxUsvLKHhpIPp1r0bzz+ziPt+/2DRZSZjz1Ejuff+Bxhz\naANVqnz35JOKLqmmZe+1PCaE0A0YA/wsxtjYeqwfcEKMcUJHTr507v3Jdrtl8a2GS4ouIXkzbptS\ndAk1ob7vxuvcVv7u+Is7nDl7nTU29zb2PTvYGGMTcMW7jr0IdChcJamrlW0GWzvTb0nJy7yTS5Ly\nYQcrSTlZn9bBStJ6pWT5asBKSocdrCTlpGT5WttP05KkPNnBSkpGVrLnNxuwkpLRWSOCEEIdMB0I\ntDx/ZSzwJi03X1WBecC4GGNzW+cpV9xL0jrIKlmHt3bsCxBj3BWYCJwBTAUmxhg/TctTBUe3dxID\nVpLeJcZ4PdDQutsfeAUYCsxuPXYzsFd75zFgJSWjM7/RIMbYFEK4EpgGXAVkMca3HyazDPhAe+cw\nYCUlI8uyDm8dEWM8GBhEyzx2za/e6ENLV9smA1ZSMiqVrMNbW0IIB4UQTmjdbQSagQdDCKNaj+0N\n3NVePa4ikKR/dB3w4xDCnUB3Wh7T+gQwPYRQ3/rzte2dxICVlIzOWqYVY1wOHPhPXhq5NucxYCUl\nw2cRSFJeSnZVyYCVlAw7WEnKScny1YCVlA47WEnKScny1YCVlJCSJWzJrrlJUjrsYCUlo1JXrg7W\ngJWUDC9ySVJOSpavzmAlKS92sJLSUbIW1oCVlIwOfNdWlzJgJSWjbAHrDFaScpJrB9tn663zPL2A\nGbdNKbqE5O00eP+iS6gJjy6c3f6/1I6SjWAdEUhKR9lGBAaspGR4o4Ek5aVc+WrASkqHHawk5cSA\nlaS8lGzhqQErKRll62BLlveSlA47WEnJcB2sJOXEgJWkvDiDlaTaYAcrKRkla2ANWEnpKNsyLQNW\nUjKyunJNPctVjSQlxA5WUjrKNSEwYCWlwxmsJOXEGw0kKSdZpVyXlcpVjSQlxA5WUjrKNSEwYCWl\no7NnsCGEYcDkGOOoEMIOwE3AgtaXL4oxzmzr/QaspHR04iqCEMJxwEHA8tZDQ4GpMcZzO3oOA1ZS\nMjp5mdZTwP7AjNb9oUAIIYympYudEGNc1tYJvMglKR2VrONbO2KMvwBWrXFoDnBsjHE34C/AKe2d\nww5WUjJyvtFgVozxlbd/Bqa19wY7WEnpyNZiW3u3hBB2bv15T+Ch9t5gByspGTl3sIcD00IIq4DF\nQEN7bzBgJek9xBifAYa3/jwX2HVt3m/ASkqHzyIon+bmZk6ffA5xwQLqu9dz2sQT2GrLLYouKzl+\nzvn4wgGfY/QBnwOgR496wrZbc/ABR3DcKeOhCk/Gpzlj0vepVqsFV5o/n0VQQrfdcScrVqzkqsun\nM2H84Zz9gx8WXVKS/JzzceO1v+Gwr07gsK9O4PF5f+asU6cx9tsHc/45l3HIl4+ADHb/7Iiiy+wS\nWZZ1eOsKaxWwIYQNQgg98iqmKHMfeYQRuwwDYMjgT/L4E/MLrihNfs752nZwYODHB/CLa37JNoMH\n8eB9DwPwhzvuZ/iIoQVXV5vaHBGEELYFvgf8H3AVcCnwVgjh2zHGm7qgvi6xfHkjG/becPV+pVJH\nU1MT3bo5QelMfs75+sb4MVx83pXAO6+mNy5vZMM+vYsqq2utZzPYi4FJwADgWmAQ8CZwMy0PPUhC\n7969WN7YuHq/udrsH/oc+Dnnp0/fDRnwsS154N4/AlBtbl79Wq/evVj22utFldalyvaNBu2NCCox\nxtkxxiuB62OML8UYXwOauqC2LrPDkO246+57AXjksXl8fODAgitKk59zfobuvB333z139f78Pz3J\nTsO3B2DEqGHMnfNoUaV1qayu0uGtK7TXPsQQwqVAQ4zxEIAQwvG0LLJNxp6jRnLv/Q8w5tAGqlT5\n7sknFV1Skvyc8zNg4FY8/+wLq/fPOf0CTjnrWLrXd+cvTy7kt7+eXWB1tStra+lGCKEC7BtjvGGN\nY2OA62KMje/5xlYrX1ua/roQJW+nwfsXXUJNeHTh7HX+/X7Jg/d2OHM22elTuc8T2uxgY4zNwA3v\nOvbTXCuSpPepbDNYrzBISocBK0n58Gu7JSkvdrCSlBMDVpLy4UUuScpLyWawPk1LknJiByspGVlW\nrp7RgJWUjLI9cNuAlZQOZ7CSVBvsYCUlw2VakpQXA1aS8pHV1RVdwjs4g5WknNjBSkqHIwJJyocX\nuSQpL97JJUn58IHbkpQXRwSSlA9nsJKUF2ewkpSTks1gyxX3kpQQO1hJyXAGK0k5ySrlehaBASsp\nHSW7yFWuaiQpIXawkpLRWXdyhRAqwIXAEGAF8I0Y45Nrex47WEnpyLKOb237ItAzxvgp4Hjg3PdT\njgErKRlZpa7DWztGAL8BiDHeB+z0furJdURQ33fjcq2ZkN6HRxfOLroEdVAnZk5f4NU19t8KIXSL\nMTatzUnsYCXpH70G9Fljv7K24QoGrCT9M3cD+wCEEIYDj72fk7iKQJL+0SzgMyGEe4AM+Pr7OUlW\nrVY7tSpJUgtHBJKUEwNWknJiwEpSTrzItYYQwjBgcoxxVNG1pCiE0B24HBgA9ABOjzHeWGhRiQkh\n1AHTgQBUgbExxnnFVlW77GBbhRCOAy4FehZdS8LGAEtjjJ8GPgecX3A9KdoXIMa4KzAROKPYcmqb\nAft3TwH7F11E4n4OTGr9OQPWeuG22hZjvB5oaN3tD7xSYDk1zxFBqxjjL0IIA4quI2UxxtcBQgh9\ngGtp6bDUyWKMTSGEK4H9gAOKrqeW2cGqS4UQtgRuB2bEGK8uup5UxRgPBgYB00MIvYuup1bZwarL\nhBD6AbcC42OMvy+6nhSFEA4Ctogxngk0As2tmwpgwKornQhsBEwKIbw9i907xvhGgTWl5jrgxyGE\nO4HuwAQ/3+J4q6wk5cQZrCTlxICVpJwYsJKUEwNWknJiwEpSTgxYScqJAStJOfl/DZxKxcrlWVwA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1186dc590>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gmm_model = GaussianMixture(n_components = 3, covariance_type = 'diag', init_params='random', max_iter=200)\n",
    "\n",
    "gmm_model.means_init = np.array([data[data['label'] == num].mean(axis = 0)[:-1] for num in range(3)])\n",
    "\n",
    "gmm_model.fit(data.iloc[:,:-1])\n",
    "\n",
    "y_pred = gmm_model.predict(data.iloc[:,:-1])\n",
    "\n",
    "\n",
    "diff = y_pred  - data['label'].values\n",
    "\n",
    "evaluate(y_pred, data['label'].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "准确率接近90%，数据集可以使用高斯混合模型来进行分类，也就是认为数据集的各个部分服从高斯分布。接下来，我们使用自定义的模型，去拟合数据集。\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先，使用数据集训练模型。我们将模型中高斯分布的个数设为`3`，迭代次数设为`2000`，终止条件的阈值设为`1e-5`。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "mus, sigmas = train_GMM(data, 3, 2000, 1e-5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`train_GMM()`函数返回各个高斯分布component的参数值。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 14.33442857,  14.29428571,   0.88007   ,   5.50805714,\n",
       "          3.24462857,   2.66740286,   5.08721429],\n",
       "       [ 18.33428571,  16.13571429,   0.88351714,   6.14802857,\n",
       "          3.67741429,   3.6448    ,   6.0206    ],\n",
       "       [ 11.87385714,  13.24785714,   0.84940857,   5.22951429,\n",
       "          2.85377143,   4.7884    ,   5.1164    ]])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 均值\n",
    "mus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[  9.61042610e-01,   4.49091020e-01,   3.64855457e-03,\n",
       "           1.56014211e-01,   1.17141015e-01,  -1.27065732e-02,\n",
       "           1.88899096e-01],\n",
       "        [  4.49091020e-01,   2.25678067e-01,  -2.32360248e-04,\n",
       "           8.79441594e-02,   4.61847713e-02,  -1.09081099e-02,\n",
       "           1.05092971e-01],\n",
       "        [  3.64855457e-03,  -2.32360248e-04,   2.53796176e-04,\n",
       "          -1.24553446e-03,   1.48731511e-03,   1.09097179e-03,\n",
       "          -1.30512355e-03],\n",
       "        [  1.56014211e-01,   8.79441594e-02,  -1.24553446e-03,\n",
       "           4.29723198e-02,   9.93728431e-03,   3.18456598e-03,\n",
       "           4.96169562e-02],\n",
       "        [  1.17141015e-01,   4.61847713e-02,   1.48731511e-03,\n",
       "           9.93728431e-03,   2.07285674e-02,   1.02206141e-03,\n",
       "           1.19468383e-02],\n",
       "        [ -1.27065732e-02,  -1.09081099e-02,   1.09097179e-03,\n",
       "           3.18456598e-03,   1.02206141e-03,   1.49197054e+00,\n",
       "           4.91593961e-02],\n",
       "        [  1.88899096e-01,   1.05092971e-01,  -1.30512355e-03,\n",
       "           4.96169562e-02,   1.19468383e-02,   4.91593961e-02,\n",
       "           8.22456169e-02]],\n",
       "\n",
       "       [[  1.57851092e+00,   6.62340210e-01,   3.56424248e-03,\n",
       "           2.56042627e-01,   1.71127913e-01,   7.63942476e-04,\n",
       "           2.28768249e-01],\n",
       "        [  6.62340210e-01,   2.95883847e-01,  -4.75500505e-04,\n",
       "           1.23587613e-01,   6.15031906e-02,   1.24584162e-02,\n",
       "           1.12445553e-01],\n",
       "        [  3.56424248e-03,  -4.75500505e-04,   2.26310981e-04,\n",
       "          -1.18668938e-03,   1.51843470e-03,  -1.93389914e-03,\n",
       "          -1.28036605e-03],\n",
       "        [  2.56042627e-01,   1.23587613e-01,  -1.18668938e-03,\n",
       "           6.34684521e-02,   1.76146078e-02,  -8.11197909e-03,\n",
       "           5.91468482e-02],\n",
       "        [  1.71127913e-01,   6.15031906e-02,   1.51843470e-03,\n",
       "           1.76146078e-02,   2.61011026e-02,   6.85939592e-03,\n",
       "           1.34256743e-02],\n",
       "        [  7.63942476e-04,   1.24584162e-02,  -1.93389914e-03,\n",
       "          -8.11197909e-03,   6.85939592e-03,   1.41923766e+00,\n",
       "          -1.50619323e-02],\n",
       "        [  2.28768249e-01,   1.12445553e-01,  -1.28036605e-03,\n",
       "           5.91468482e-02,   1.34256743e-02,  -1.50619323e-02,\n",
       "           6.38161865e-02]],\n",
       "\n",
       "       [[  4.85828865e-01,   2.15529230e-01,   6.98968287e-03,\n",
       "           5.38062306e-02,   8.22069311e-02,   4.78245130e-02,\n",
       "           3.23371835e-02],\n",
       "        [  2.15529230e-01,   1.16762669e-01,   3.92567483e-04,\n",
       "           3.96924548e-02,   2.63491802e-02,   4.63964515e-02,\n",
       "           3.34474027e-02],\n",
       "        [  6.98968287e-03,   3.92567483e-04,   4.48495788e-04,\n",
       "          -1.26743868e-03,   2.46968621e-03,  -3.04704732e-03,\n",
       "          -2.00181745e-03],\n",
       "        [  5.38062306e-02,   3.96924548e-02,  -1.26743868e-03,\n",
       "           2.01866081e-02,   1.52143114e-03,   3.86411654e-02,\n",
       "           2.06172222e-02],\n",
       "        [  8.22069311e-02,   2.63491802e-02,   2.46968621e-03,\n",
       "           1.52143114e-03,   1.94984539e-02,   1.04909818e-02,\n",
       "          -3.64911722e-03],\n",
       "        [  4.78245130e-02,   4.63964515e-02,  -3.04704732e-03,\n",
       "           3.86411654e-02,   1.04909818e-02,   1.94417811e+00,\n",
       "           5.69210636e-02],\n",
       "        [  3.23371835e-02,   3.34474027e-02,  -2.00181745e-03,\n",
       "           2.06172222e-02,  -3.64911722e-03,   5.69210636e-02,\n",
       "           3.19963703e-02]]])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 协方差矩阵\n",
    "sigmas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "预测数据集的标签。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  2.,  0.,  0.,  0.,  2.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  2.,  2.,  2.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  2.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "        1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "        1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "        1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "        1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "        1.,  1.,  0.,  1.,  0.,  0.,  1.,  0.,  0.,  1.,  2.,  2.,  2.,\n",
       "        2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,\n",
       "        2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,\n",
       "        2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,\n",
       "        2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,\n",
       "        2.,  2.,  0.,  2.,  2.,  2.,  0.,  2.,  2.,  2.,  2.,  2.,  2.,\n",
       "        2.,  2.])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_pred = predict(data, mus, sigmas)\n",
    "label_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9286\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.90      0.89      0.89        70\n",
      "          1       0.97      0.93      0.95        70\n",
      "          2       0.92      0.97      0.94        70\n",
      "\n",
      "avg / total       0.93      0.93      0.93       210\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD3CAYAAABYUUzPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE5dJREFUeJzt3Xm8XeO9x/HP3icnoySo0hJDr9ZzzdGEqCAh5ouQqva2\nFFExxNTcSIVEuGYRU4Q2MU+tmlXbq14toaWGahHD06ihapbIIIecc3L2/eMcrms45yQ5a6/lOZ+3\n13q9zl47+/Hjj29+57eetXapUqkgSep45bwLkKRUGbCSlBEDVpIyYsBKUkYMWEnKSJcsF995o33d\nopCx2+89P+8Sktcwf17eJXQKfb6+YWl519hk7SHtzpwnX5653P++ttjBSlJGMu1gJamaSqXMm9Kl\nYsBKSkapVKxfyg1YSckoYwcrSZlwRCBJGSk7IpCkbBStgy1W3EtSQuxgJSWjplSTdwn/jwErKRlF\nGxEYsJKSUS5YwDqDlaSM2MFKSkapYD2jASspGTVlA1aSMlEq2K2yxYp7SUqIHaykZHirrCRlxH2w\nkpSRou2DNWAlJaNoF7kMWEnJcAYrSRlxBitJGenIGWwIYTywJ9AVuASYCVwFVIBZwOgYY1Or9XRY\nNZKUs9JS/NOaEMJQYCtgMDAEWBM4D5gQY9wGKAHD26rHgJWUjFKp1O6jDTsDTwG3Ab8C7gIG0NzF\nAvwW2KGtRRwRSEpGB44IVgHWBnYHvgbcCZRjjJWW9xcCfdtaxICVlIwO3KY1B3guxlgPxBDCBzSP\nCT7UG5jX1iKOCCQlo1wqt/towx+BXUIIpRDC6kAv4Pcts1mAXYEH2lrEDlaSPiHGeFcIYVvgEZob\n0dHAi8CMEEJX4Fng5rbWMWAlJaMj98HGGMd9xukhS7OGASspGTXeySVJ2fBhLwXz3R/txZZDB1Jb\n24Vf3Xg3s59+gdEnjGRJUxMN9Q1MPmEa8+bMz7vMJDQ0NnLymefw2utvUt9QzyEH7MfQrQfnXVZy\nrvzlLTzw8KM0NDSyz3/swvCd29yuqYx06oDdZPMN2KB/YMz+E+nWvSv7HLQnO+wxhGlnXMEL8WV2\n+84O7DtyONMnX5N3qUn49d330LdPH06feALzFyzguweOMmA72F+enMWTz0Yum3wGHyxezHW33pF3\nSVXlswgKZMDgTXlp9j+ZdOFYeq7QgxlTruM3v7yHue80b2+rqamhob4h5yrTsdN2Q9lxu+ZrBJVK\nhZqampwrSs9Dj/+Nr6+zFseddjaL6t7n6IN/mHdJVeWIoED6rtiHVVdfhZOOOIuv9FuVk6f+hB/t\ncSwAG/Rfjz2/vzNjD5iUc5Xp6NmzBwCL6uoYO+EURh8yMueK0jNvwQLeeOttzp90Aq+++Rb/9d9n\ncvPPphaus8uKz4MtkAXzFvLKi6/S2LiEf730Og319fRduQ/9t9iQ740awcQjzmL+uwvzLjMpb7z5\nFmNOOIl99x7ObjsNy7uc5PTt3Zt1+q1BbW0t6/Rbg25da3l3/nxWXnHFvEurii9UBxtCuBfo9onT\nJaASY9wqs6qq5Om/Psde++3GLVffxcpfXonuPbqz+dabseu3t2fcgSezcMGivEtMypy5czl8zDiO\n//HRDBr4zbzLSVL/DdfnF3fcxQ/23pN35r7L+x8spm/v3nmXVTVF69Tb6mCPB2YAewON2ZdTXQ/P\nfJyNBqzPRb84g3KpzMWnXc74ycfw1uvvMPHCsQA89dgzXDvtppwrTcNl19zAgoULmX7VtUy/6loA\npk05i+7dPvl3uJbVNlsM5K+znuGAH4+j0lRh3OGHdKpZd9E62FKlUmn1D4QQjgOejzHetrSL77zR\nvq0vruV2+73n511C8hrmt/lMD3WAPl/fcLnT8eCtRrc7cy5/cFrmadzmDDbGODnrIiSpIxStg+3U\nF7kkpeWLNoOVpC8MO1hJyogdrCRlpGg3GhTr2V6SlBA7WEnJKBergTVgJaWjplysX8oNWEnJKNpF\nrmLFvSQlxA5WUjLKBdtFYMBKSkbRRgQGrKRkeCeXJGWkYPlqwEpKhx2sJGWkaLfKGrCSkuFFLknK\niCMCScpIwfLVgJWUDjtYScqIF7kkKSN2sJKUkYLlq0/TkqSs2MFKSoYP3JakjBRtRGDASkpG0S5y\nFauflqSE2MFKSob7YCUpIz7sRZIyUlPu2IANIawK/AXYEegB3AXMbnn70hjjja193oCVpM8QQqgF\nfga833JqAHBejHFKe9fINGBvvuvULJcXMGzwqLxLSN59j12Tdwlqpw4eEZwL/BQY3/J6ABBCCMNp\n7mKPjTEubG0BdxFISka51P6jNSGEA4G3Y4x3f+z0I8BxMcZtgReASW3V44hAUjI6sIMdCVRCCDsA\n/YFrgD1jjG+0vH8bMLWtRQxYScnoqHxt6VIBCCHcBxwG3BFCOCrG+AgwjOaLX60yYCUlI+M7uQ4H\npoYQGoA3gDYvgBiwkpKRxY0GMcahH3s5eGk+a8BKSkbB7jMwYCWlw4e9SFInYQcrKRnlDr5VdnkZ\nsJKS4cNeJCkjBWtgncFKUlbsYCUlwxGBJGWko58Hu7wMWEnJKFoH6wxWkjJiByspGQVrYA1YSeko\n2ojAgJWUjILlqwErKR1Fe9iLASspGQXLVwNWUjqcwUpSRgqWrwaspHQUrYP1RgNJyogdrKRk+CwC\nScpIwSYEBqykdDiDlaROwg5WUjIK1sAasJLS4bfKFtQPRh9Lr549AVhjtdWYNPaYnCtKx/6Hfodt\nth9El9pabr3h1/z96eeZPH0Sr7z0GgC3/fw3/P43D+RcZRqampo47exzibNn07W2K6dMGM9aa/bL\nu6yqKdoM1oAFFtfXU6lUmD75jLxLSc5mW2zMxputz6HfO47uPbrx/YNHUCrBL668nZ9fcVve5SXn\nD/fdz+LF9Vx/xQyeeGoWky+4iKlTzsm7rE5rqQM2hNAtxrg4i2LyMvuFF/lgcT2jx5/EkiVLGH3Q\n/my8/r/nXVYSBm3zTf7x95c4a9oEeq7Qk2nnXM7u++zE2l/rxzbDtuSVl17jwjOmU7fo/bxLTcLj\nTzzB1lsNAmDTjTfimWefy7mi6ipYA/v5ARtC2AO4GGgATowx3tjy1m+B7atQW9V079aN/b+9F3vt\nuhP/fPU1jp5wCrdcfildamryLu0Lb8WV+vCV1Vdl7KGnsHq/1Tj70pO4bvpN/Oqm3xGffp4DDvsu\nI4/8PheffXnepSZh0aI6Vui1wkevy+UaGhsb6dKlc/yyWrQRQWvbtE4E+gODgENDCAe0nC/Wf0EH\nWGuNNdh12FBKpRJr91uDvr17886cuXmXlYT58xby8B8fp7GhkX+++Cr19fU8eN+jxKefB2DmPQ+y\n3gb/lnOV6ejVqyeL6uo+et1Uaeo04QrNHWx7j2poLWDrY4zvxhjnAMOBI0MI2wGV6pRWPXfefQ8X\nTL8CgLfnzGFRXR2rfGnlnKtKwxOPPc2gbQYAsMqqK9OjR3fOnXEy62+yHgADt+rPc7Oez7PEpGy2\n6SY88KeHAHjiqVl8Y911c66oumrKpXYf1dDaX20vhRDOAybGGBeGEEYAdwMrVqWyKhq+y46cfO6F\nHDzmJ5QocdKYox0PdJAH73uU/ptvxOW3nE+pVGbKKZfy7tz5jJl4GI2Njcx9513OmjA17zKTMWzo\nEB56+FH2GzmKChVOPenEvEvq1FoL2JHAfrR0rDHGV1o62PHVKKyaamtrOX382LzLSNYlk6/81LnD\n/vO4HCpJX7lc5qTx4/IuIzdFm8F+bsDGGBuBqz5x7k3g2IxrkqRlUrB8dR+spHSUvJNLkrJhBytJ\nGfnCzGAl6YumYPlqwEpKhx2sJGWko/I1hFADzAACzVtVDwM+oHlnVQWYBYyOMTa1to7faCBJn7YH\nQIxxMDABOB04D5gQY9yG5kcGDG9rEQNWUjJK5XK7j9bEGG8HRrW8XBuYBwwAZrac+y2wQ1v1GLCS\nktGRD3uJMTaGEK4GpgLXA6UY44fPYlkI9G1rDQNWUjJK5VK7j/aIMR4ArEfzPLbHx97qTXNX2yoD\nVpI+IYSwfwjhw+eu1AFNwGMhhKEt53YF2vyeI3cRSEpGB+7SuhW4MoRwP1BL8zNYngVmhBC6tvx8\nc1uLGLCSktFR+2BjjIuAfT/jrSFLs44BKykZRfvabmewkpQRO1hJySjYnbIGrKR0+CwCScpKwYae\nBqykZNjBSlJGCpavBqykdNjBSlJGCpavBqykhBQsYQt2zU2S0mEHKykZ5ZpidbAGrKRkeJFLkjJS\nsHx1BitJWbGDlZSOgrWwBqykZLT3u7aqxYCVlIyiBawzWEnKSKYdbJdeK2S5vID7Hrsm7xKSN3Dj\nEXmX0Ck8+fLM5V6jYCNYRwSS0lG0EYEBKykZ3mggSVkpVr4asJLSYQcrSRkxYCUpKwXbeGrASkpG\n0TrYguW9JKXDDlZSMtwHK0kZMWAlKSvOYCWpc7CDlZSMgjWwBqykdBRtm5YBKykZpZpiTT2LVY0k\nJcQOVlI6ijUhMGAlpcMZrCRlxBsNJCkjpXKxLisZsJL0OUIIg4CzY4xDQwibAXcBs1vevjTGeGNr\nnzdgJaWjAycEIYRxwP7AopZTA4DzYoxT2ruGASspGR08g/0HMAK4tuX1ACCEEIbT3MUeG2Nc2NoC\nxRpYSNLyKJXaf7QhxngL0PCxU48Ax8UYtwVeACa1tYYdrKRkZLxN67YY47wPfwamtvUBO1hJ6SiX\n2n8svbtDCFu0/DwM+EtbH7CDlZSMjDvYw4GpIYQG4A1gVFsfMGAlpaOD8zXG+BKwZcvPjwODl+bz\nBqykZBTtVllnsJKUETtYSenwWQTF09DYyMlnnsNrr79JfUM9hxywH0O3XqpRi9qhqamJ084+lzh7\nNl1ru3LKhPGstWa/vMtKwsFH/IChOw6mtrYLN157O8889Xcmnj6GxiVLePnFf3HyuHOoVCp5l5m5\noj2LoFjV5OTXd99D3z59uPKSC7lkytmcdV6b29u0DP5w3/0sXlzP9VfM4NgjD2fyBRflXVISBm7Z\nn/4DNuKHI0Zz0L7H8JWvrsphxxzITy+6mgP3OYquXWvZdvtv5V1mVZRKpXYf1bBUHWwIoQfQFGNc\nnFE9udhpu6HsuN0QACqVCjU1NTlXlKbHn3iCrbcaBMCmG2/EM88+l3NFadhq282Z/dwLXDD9NHr1\n7sV5p19KU1OFviv2AaBXr540NjbmXGXn1GrAhhA2AM4A3gWuBy4DloQQjokx3lWF+qqiZ88eACyq\nq2PshFMYfcjInCtK06JFdazQa4WPXpfLNTQ2NtKli5Oq5bHSyivy1TVW48iRx9Nvza9y0WVncOkF\nV3HCqccy6sj9eW/hIh7989/yLrM6CjaDbWtE8FPgfOA+4GZgC2AzYHy2ZVXfG2++xSFHjWH3nXdk\nt52G5V1Oknr16smiurqPXjdVmgzXDjDv3fk8eP8jNDY08tILr7B4cT1nXjiBA/c5iuHDfsidt97N\n2AlH5F1mVRRtRNBWwJZjjDNjjFcDt8cY34oxLgCS+n1jzty5HD5mHMccPoq9dt8173KStdmmm/DA\nnx4C4ImnZvGNddfNuaI0/PXRpxg8pHn08uVVv0SPnt155eXXeO+95qfsvf3mHPr07Z1niVVTqim3\n+6iGttqHGEK4DBgVYzwQIIRwPM23iSXjsmtuYMHChUy/6lqmX9X8ZLJpU86ie7duOVeWlmFDh/DQ\nw4+y38hRVKhw6kkn5l1SEu7/w0MMGLQpN9z5M8rlEmdMvID36z7gnIsnsaRxCQ0NjZxy/OS8y+yU\nSq1t3QghlIE9Yox3fOzcfsCtMca6z/1gi/fffjX9fSE5q+nWPe8Skjdw4xF5l9ApPPnyzOX+vf2d\nxx5qd+asMvBbmc8JWu1gY4xNwB2fOHddphVJ0jIq2q2yXmGQlA4DVpKy4dd2S1JW7GAlKSMGrCRl\nw4tckpSVgs1gfZqWJGXEDlZSMkqlYvWMBqykZBTtgdsGrKR0OIOVpM7BDlZSMtymJUlZMWAlKRul\ngn2fnjNYScqIHaykdDgikKRseJFLkrLinVySlA0fuC1JWXFEIEnZcAYrSVlxBitJGSnYDLZYcS9J\nCbGDlZQMZ7CSlJFSuVjPIjBgJaWjYBe5ilWNJCXEDlZSMjrqTq4QQhm4BNgUWAz8KMb4/NKuYwcr\nKR2lUvuP1u0FdI8xfgs4HpiyLOUYsJKSUSrXtPtow9bA/wDEGP8MDFyWejIdEfT48hrF2jMhLYMn\nX56Zdwlqp659vtRRmdMHmP+x10tCCF1ijI1Ls4gdrCR92gKg98del5c2XMGAlaTP8idgN4AQwpbA\nU8uyiLsIJOnTbgN2DCE8CJSAg5ZlkVKlUunQqiRJzRwRSFJGDFhJyogBK0kZ8SLXx4QQBgFnxxiH\n5l1LikIItcAVwDpAN+C0GOOduRaVmBBCDTADCEAFOCzGOCvfqjovO9gWIYRxwGVA97xrSdh+wJwY\n4zbALsDFOdeToj0AYoyDgQnA6fmW07kZsP/nH8CIvItI3E3AxJafS8BSb9xW62KMtwOjWl6uDczL\nsZxOzxFBixjjLSGEdfKuI2UxxvcAQgi9gZtp7rDUwWKMjSGEq4G9gX3yrqczs4NVVYUQ1gTuBa6N\nMd6Qdz2pijEeAKwHzAgh9Mq7ns7KDlZVE0JYDfgdcGSM8fd515OiEML+QL8Y45lAHdDUcigHBqyq\n6QRgJWBiCOHDWeyuMcb3c6wpNbcCV4YQ7gdqgWP9/5sfb5WVpIw4g5WkjBiwkpQRA1aSMmLASlJG\nDFhJyogBK0kZMWAlKSP/Cy1heZ+erYLpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x118875c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(label_pred, data['label'].values)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
